diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 2b7fafa25..3b5a1157d 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -213,9 +213,7 @@ test('test command --verbose prints step telemetry for passing tests without deb }); test('test command --verbose keeps nested retry and open step telemetry distinct', async () => { - const tmpDir = await fs.mkdtemp( - path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'), - ); + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-')); const artifactsDir = path.join(tmpDir, 'material-top-tabs'); const attemptDir = path.join(artifactsDir, 'attempt-1'); await fs.mkdir(attemptDir, { recursive: true }); @@ -497,6 +495,32 @@ test('test --maestro forwards Maestro backend and platform for directory suites' } }); +test('test forwards shard flags and comma device lists', async () => { + const result = await runCliCapture( + ['test', '--maestro', '--device', 'udid1,emulator-5554', '--shard-all', '2', './suite'], + async () => ({ + ok: true, + data: { + total: 0, + executed: 0, + passed: 0, + failed: 0, + skipped: 0, + notRun: 0, + durationMs: 1, + failures: [], + tests: [], + }, + }), + ); + + assert.equal(result.code, null); + assert.equal(result.calls.length, 1); + assert.equal(result.calls[0]?.flags?.replayBackend, 'maestro'); + assert.equal(result.calls[0]?.flags?.device, 'udid1,emulator-5554'); + assert.equal(result.calls[0]?.flags?.shardAll, 2); +}); + test('test command writes JUnit report with failure metadata', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-junit-test-')); const reportPath = path.join(tmpDir, 'replays.junit.xml'); diff --git a/src/cli-test.ts b/src/cli-test.ts index d1b9240c3..ee21fc3e8 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -317,8 +317,8 @@ function renderFlakyTestSummary( function replayTestDisplayName(result: ReplaySuiteTestResult): string { const title = replayTestTitle(result); - if (title && title.length > 0) return JSON.stringify(title); - return path.basename(result.file); + const base = title && title.length > 0 ? JSON.stringify(title) : path.basename(result.file); + return `${base}${formatReplayTestShardSuffix(result)}`; } function replayFailedTestDisplayName( @@ -326,11 +326,12 @@ function replayFailedTestDisplayName( ): string { const title = replayTestTitle(result); const filename = path.basename(result.file); - return title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; + const base = title && title.length > 0 ? `${JSON.stringify(title)} in ${filename}` : filename; + return `${base}${formatReplayTestShardSuffix(result)}`; } function replayTestCaseName(result: ReplaySuiteTestResult): string { - return replayTestTitle(result) ?? path.basename(result.file); + return `${replayTestTitle(result) ?? path.basename(result.file)}${formatReplayTestShardSuffix(result)}`; } function replayTestTitle(result: ReplaySuiteTestResult): string | undefined { @@ -402,7 +403,7 @@ function buildReplayJunitXml(suite: ReplaySuiteResult): string { function renderJUnitTestCase(test: ReplaySuiteTestResult): string[] { const name = xmlEscape(replayTestCaseName(test)); const className = xmlEscape( - path.dirname(test.file) === '.' ? test.file : path.dirname(test.file), + `${path.dirname(test.file) === '.' ? test.file : path.dirname(test.file)}${formatReplayTestShardSuffix(test)}`, ); const file = xmlEscape(test.file); const time = formatJUnitSeconds(test.durationMs); @@ -450,12 +451,33 @@ function appendReplaySystemOutMetadata(lines: string[], test: ReplaySuiteTestRes lines, 'artifactsDir' in test && test.artifactsDir ? `artifactsDir: ${test.artifactsDir}` : undefined, ); + appendReplayTestShardMetadata(lines, test); if (test.status === 'failed') { appendReplayFailureSystemOut(lines, test); } appendOptionalLine(lines, isFlakyReplayTestResult(test) ? 'flaky: true' : undefined); } +function formatReplayTestShardSuffix(result: ReplaySuiteTestResult): string { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return ''; + const shardCount = typeof result.shardCount === 'number' ? result.shardCount : '?'; + const device = typeof result.deviceId === 'string' ? ` ${result.deviceId}` : ''; + return ` [shard ${result.shardIndex + 1}/${shardCount}${device}]`; +} + +function appendReplayTestShardMetadata(lines: string[], result: ReplaySuiteTestResult): void { + if (!('shardIndex' in result) || typeof result.shardIndex !== 'number') return; + lines.push(`shardIndex: ${result.shardIndex}`); + appendOptionalLine( + lines, + typeof result.shardCount === 'number' ? `shardCount: ${result.shardCount}` : undefined, + ); + appendOptionalLine( + lines, + typeof result.deviceId === 'string' ? `deviceId: ${result.deviceId}` : undefined, + ); +} + function appendReplayFailureSystemOut( lines: string[], test: Extract, diff --git a/src/client-normalizers.ts b/src/client-normalizers.ts index 76ae05af0..7ab982446 100644 --- a/src/client-normalizers.ts +++ b/src/client-normalizers.ts @@ -321,6 +321,8 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { retries: options.retries, artifactsDir: options.artifactsDir, reportJunit: options.reportJunit, + shardAll: options.shardAll, + shardSplit: options.shardSplit, findFirst: options.findFirst, findLast: options.findLast, networkInclude: options.networkInclude, diff --git a/src/client-types.ts b/src/client-types.ts index 8302cbcc9..02f339bfa 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -718,6 +718,8 @@ export type ReplayTestOptions = AgentDeviceRequestOverrides & retries?: number; artifactsDir?: string; reportJunit?: string; + shardAll?: number; + shardSplit?: number; }; export type BatchStep = { @@ -855,6 +857,8 @@ type CommandExecutionOptions = Partial & { retries?: number; artifactsDir?: string; reportJunit?: string; + shardAll?: number; + shardSplit?: number; findFirst?: boolean; findLast?: boolean; networkInclude?: 'summary' | 'headers' | 'body' | 'all'; diff --git a/src/commands/cli-grammar/replay.ts b/src/commands/cli-grammar/replay.ts index 2cdfd5d6d..335561ec6 100644 --- a/src/commands/cli-grammar/replay.ts +++ b/src/commands/cli-grammar/replay.ts @@ -21,6 +21,8 @@ export const replayCliReaders = { retries: flags.retries, artifactsDir: flags.artifactsDir, reportJunit: flags.reportJunit, + shardAll: flags.shardAll, + shardSplit: flags.shardSplit, }), } satisfies Record; diff --git a/src/commands/client-command-metadata.ts b/src/commands/client-command-metadata.ts index 35d32fe0a..dbf903b6d 100644 --- a/src/commands/client-command-metadata.ts +++ b/src/commands/client-command-metadata.ts @@ -177,6 +177,8 @@ export const clientCommandMetadata = [ retries: integerField(), artifactsDir: stringField(), reportJunit: stringField(), + shardAll: integerField(), + shardSplit: integerField(), }), defineClientCommandMetadata('perf', { area: enumField(PERF_AREA_VALUES), diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index f044b3920..6f23139ae 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -22,6 +22,8 @@ export type CommandFlags = Omit & { maestro?: MaestroRuntimeFlags; postGestureStabilization?: boolean; replayBackend?: string; + shardCount?: number; + shardIndex?: number; }; export type DispatchContext = ScreenshotDispatchFlags & { diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 9fd5a06d1..8b3cbe5fa 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -6,6 +6,8 @@ import { handleSessionCommands } from '../session.ts'; import { SessionStore } from '../../session-store.ts'; import type { DaemonRequest, DaemonResponse, DaemonResponseData } from '../../types.ts'; import { type RequestProgressEvent, withRequestProgressSink } from '../../request-progress.ts'; +import { withDeviceInventoryProvider } from '../../../core/dispatch-resolve.ts'; +import type { DeviceInfo } from '../../../utils/device.ts'; function makeSessionStore(): SessionStore { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-session-test-suite-')); @@ -13,11 +15,27 @@ function makeSessionStore(): SessionStore { } function expectOkData(response: DaemonResponse | null | undefined): DaemonResponseData { - expect(response?.ok).toBeTruthy(); + expect(response?.ok, JSON.stringify(response)).toBeTruthy(); if (!response || !response.ok) throw new Error('Expected successful daemon response.'); return response.data ?? {}; } +const ANDROID_ONE: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel 8', + kind: 'emulator', + booted: true, +}; + +const ANDROID_TWO: DeviceInfo = { + platform: 'android', + id: 'emulator-5556', + name: 'Pixel 8 Pro', + kind: 'emulator', + booted: true, +}; + test('test does not retry infrastructure startup failures and stops the suite', async () => { const sessionStore = makeSessionStore(); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-infra-fail-')); @@ -200,3 +218,235 @@ test('test emits skip progress without synthetic duration', async () => { }); expect(events[0]?.durationMs).toBeUndefined(); }); + +test('test --shard-all runs each runnable entry on each selected device', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-all-')); + fs.writeFileSync(path.join(root, '01-login.ad'), 'context platform=android\nopen "Demo"\n'); + fs.writeFileSync(path.join(root, '02-pay.ad'), 'context platform=android\nopen "Demo"\n'); + + const invoked: DaemonRequest[] = []; + const response = await withDeviceInventoryProvider( + async () => [ANDROID_ONE, ANDROID_TWO], + async () => + await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-all' }, + flags: { + platform: 'android', + device: 'emulator-5554,emulator-5556', + shardAll: 2, + }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async (req) => { + invoked.push(req); + return { ok: true, data: { replayed: 1, healed: 0 } }; + }, + }), + ); + + const data = expectOkData(response); + expect(data.total).toBe(4); + expect(data.passed).toBe(4); + expect(invoked.map((req) => req.flags?.serial).sort()).toEqual([ + 'emulator-5554', + 'emulator-5554', + 'emulator-5556', + 'emulator-5556', + ]); + const tests = data.tests as Array>; + expect(tests.map((entry) => entry.deviceId)).toEqual([ + 'emulator-5554', + 'emulator-5554', + 'emulator-5556', + 'emulator-5556', + ]); + expect(tests.map((entry) => entry.artifactsDir)).toEqual([ + expect.stringContaining(`${path.sep}shard-1${path.sep}01-login`), + expect.stringContaining(`${path.sep}shard-1${path.sep}02-pay`), + expect.stringContaining(`${path.sep}shard-2${path.sep}01-login`), + expect.stringContaining(`${path.sep}shard-2${path.sep}02-pay`), + ]); + expect(tests.map((entry) => String(entry.session))).toEqual([ + expect.stringContaining('default:shard-1:test:'), + expect.stringContaining('default:shard-1:test:'), + expect.stringContaining('default:shard-2:test:'), + expect.stringContaining('default:shard-2:test:'), + ]); +}); + +test('test --shard-split distributes runnable entries by modulo and keeps skips once', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-split-')); + fs.writeFileSync(path.join(root, '01-missing-platform.ad'), 'open "Demo"\n'); + fs.writeFileSync(path.join(root, '02-a.ad'), 'context platform=android\nopen "Demo"\n'); + fs.writeFileSync(path.join(root, '03-b.ad'), 'context platform=android\nopen "Demo"\n'); + fs.writeFileSync(path.join(root, '04-c.ad'), 'context platform=android\nopen "Demo"\n'); + + const invoked: DaemonRequest[] = []; + const response = await withDeviceInventoryProvider( + async () => [ANDROID_TWO, ANDROID_ONE], + async () => + await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-split' }, + flags: { platform: 'android', shardSplit: 2 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async (req) => { + invoked.push(req); + return { ok: true, data: { replayed: 1, healed: 0 } }; + }, + }), + ); + + const data = expectOkData(response); + expect(data.total).toBe(4); + expect(data.skipped).toBe(1); + expect(data.passed).toBe(3); + const tests = data.tests as Array>; + expect( + tests + .filter((entry) => entry.status === 'passed') + .map((entry) => path.basename(String(entry.file))), + ).toEqual(['02-a.ad', '04-c.ad', '03-b.ad']); + expect(tests.filter((entry) => entry.status === 'passed').map((entry) => entry.deviceId)).toEqual( + ['emulator-5554', 'emulator-5554', 'emulator-5556'], + ); + expect(invoked.map((req) => req.flags?.serial).sort()).toEqual([ + 'emulator-5554', + 'emulator-5554', + 'emulator-5556', + ]); + expect(tests.filter((entry) => entry.status === 'skipped')).toHaveLength(1); +}); + +test('test sharding rejects mutually exclusive shard modes', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-modes-')); + fs.writeFileSync(path.join(root, '01-a.ad'), 'context platform=android\nopen "Demo"\n'); + + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-modes' }, + flags: { platform: 'android', shardAll: 2, shardSplit: 2 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + }); + + expect(response?.ok).toBe(false); + if (response?.ok !== false) throw new Error('Expected failed daemon response.'); + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.message).toMatch(/mutually exclusive/); +}); + +test('test sharding rejects non-positive shard counts', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-count-')); + fs.writeFileSync(path.join(root, '01-a.ad'), 'context platform=android\nopen "Demo"\n'); + + const response = await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-count' }, + flags: { platform: 'android', shardAll: 0 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + }); + + expect(response?.ok).toBe(false); + if (response?.ok !== false) throw new Error('Expected failed daemon response.'); + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.message).toMatch(/positive integer/); +}); + +test('test sharding rejects fewer matched devices than requested shards', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-devices-')); + fs.writeFileSync(path.join(root, '01-a.ad'), 'context platform=android\nopen "Demo"\n'); + + const response = await withDeviceInventoryProvider( + async () => [ANDROID_ONE], + async () => + await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-devices' }, + flags: { platform: 'android', shardAll: 2 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + }), + ); + + expect(response?.ok).toBe(false); + if (response?.ok !== false) throw new Error('Expected failed daemon response.'); + expect(response.error.code).toBe('DEVICE_NOT_FOUND'); + expect(response.error.message).toMatch(/requires 2 devices, but only 1 matched/); +}); + +test('test sharding does not require devices when every entry is skipped', async () => { + const sessionStore = makeSessionStore(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-suite-shard-skipped-')); + fs.writeFileSync(path.join(root, '01-missing-platform.ad'), 'open "Demo"\n'); + + let inventoryResolved = false; + const response = await withDeviceInventoryProvider( + async () => { + inventoryResolved = true; + return [ANDROID_ONE, ANDROID_TWO]; + }, + async () => + await handleSessionCommands({ + req: { + token: 't', + session: 'default', + command: 'test', + positionals: [root], + meta: { cwd: root, requestId: 'suite-shard-skipped' }, + flags: { platform: 'android', shardAll: 2 }, + }, + sessionName: 'default', + logPath: path.join(os.tmpdir(), 'daemon.log'), + sessionStore, + invoke: async () => ({ ok: true, data: { replayed: 1, healed: 0 } }), + }), + ); + + expect(response?.ok).toBe(false); + if (response?.ok !== false) throw new Error('Expected failed daemon response.'); + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.message).toBe('No replay tests matched for --platform android.'); + expect(inventoryResolved).toBe(false); +}); diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 4e95da291..0e3dbf70c 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -180,6 +180,15 @@ function buildReplayBuiltinVars(params: { if (target) builtins.AD_TARGET = target; const device = flags.device; if (typeof device === 'string' && device.length > 0) builtins.AD_DEVICE = device; + const deviceId = typeof flags.serial === 'string' ? flags.serial : flags.udid; + if (typeof deviceId === 'string' && deviceId.length > 0) { + builtins.AD_DEVICE_ID = deviceId; + } + if (typeof flags.shardIndex === 'number') { + const shardIndex = String(flags.shardIndex); + builtins.AD_SHARD_INDEX = shardIndex; + } + if (typeof flags.shardCount === 'number') builtins.AD_SHARD_COUNT = String(flags.shardCount); const artifactsDir = flags.artifactsDir; if (typeof artifactsDir === 'string' && artifactsDir.length > 0) { builtins.AD_ARTIFACTS = artifactsDir; diff --git a/src/daemon/handlers/session-replay.ts b/src/daemon/handlers/session-replay.ts index 58567d492..f04f87da2 100644 --- a/src/daemon/handlers/session-replay.ts +++ b/src/daemon/handlers/session-replay.ts @@ -5,23 +5,33 @@ import { runReplayTestSuite } from './session-test.ts'; import { handleCloseCommand } from './session-close.ts'; import { collectReplayActionArtifactPaths, runReplayScriptFile } from './session-replay-runtime.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; +import { buildReplayTestShardFlags, type ReplayTestShardContext } from './session-test-sharding.ts'; export function buildNestedReplayFlags(params: { parentFlags: CommandFlags | undefined; platform: ReplayScriptMetadata['platform'] | undefined; target: ReplayScriptMetadata['target'] | undefined; artifactsDir: string | undefined; + shard?: ReplayTestShardContext; }): CommandFlags | undefined { - const { parentFlags, platform, target, artifactsDir } = params; - if (platform === undefined && target === undefined && artifactsDir === undefined) { + const { parentFlags, platform, target, artifactsDir, shard } = params; + if ( + platform === undefined && + target === undefined && + artifactsDir === undefined && + shard === undefined + ) { return parentFlags; } - return { - ...(parentFlags ?? {}), - ...(platform !== undefined ? { platform } : {}), - ...(target !== undefined ? { target } : {}), - ...(artifactsDir !== undefined ? { artifactsDir } : {}), - }; + return buildReplayTestShardFlags( + { + ...(parentFlags ?? {}), + ...(platform !== undefined ? { platform } : {}), + ...(target !== undefined ? { target } : {}), + ...(artifactsDir !== undefined ? { artifactsDir } : {}), + }, + shard, + ); } export async function handleSessionReplayCommands(params: { @@ -56,6 +66,7 @@ export async function handleSessionReplayCommands(params: { artifactsDir, artifactPaths, tracePath, + shard, }) => { const captureArtifacts = (response: DaemonResponse): DaemonResponse => { if (!artifactPaths) return response; @@ -68,6 +79,7 @@ export async function handleSessionReplayCommands(params: { platform, target, artifactsDir, + shard, }); return await runReplayScriptFile({ diff --git a/src/daemon/handlers/session-test-discovery.ts b/src/daemon/handlers/session-test-discovery.ts index 6032559d9..a0f3a3381 100644 --- a/src/daemon/handlers/session-test-discovery.ts +++ b/src/daemon/handlers/session-test-discovery.ts @@ -108,10 +108,12 @@ export function buildReplayTestAttemptRequestId(params: { filePath: string; caseIndex: number; attemptIndex: number; + shardIndex?: number; }): string { - const { requestId, suiteInvocationId, filePath, caseIndex, attemptIndex } = params; + const { requestId, suiteInvocationId, filePath, caseIndex, attemptIndex, shardIndex } = params; + const shardPart = shardIndex === undefined ? '' : `:shard:${shardIndex + 1}`; return resolveRequestTrackingId( - `${requestId ?? suiteInvocationId}:test:${caseIndex + 1}:${path.basename(filePath)}:attempt:${attemptIndex + 1}`, + `${requestId ?? suiteInvocationId}${shardPart}:test:${caseIndex + 1}:${path.basename(filePath)}:attempt:${attemptIndex + 1}`, suiteInvocationId, ); } diff --git a/src/daemon/handlers/session-test-runtime.ts b/src/daemon/handlers/session-test-runtime.ts index f0d4015c8..7591be1b5 100644 --- a/src/daemon/handlers/session-test-runtime.ts +++ b/src/daemon/handlers/session-test-runtime.ts @@ -10,7 +10,10 @@ import { } from '../request-cancel.ts'; import type { DaemonResponse } from '../types.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; -import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; +import type { + ReplayTestRunReplayParams, + ReplayTestRuntimeDependencies, +} from './session-test-types.ts'; const REPLAY_TIMEOUT_CLEANUP_GRACE_MS = 2_000; const REPLAY_TEST_TIMEOUT_HINT = @@ -26,6 +29,7 @@ export async function runReplayTestAttempt( platform?: ReplayScriptMetadata['platform']; target?: ReplayScriptMetadata['target']; artifactsDir?: string; + shard?: ReplayTestRunReplayParams['shard']; } & ReplayTestRuntimeDependencies, ): Promise { const { @@ -36,6 +40,7 @@ export async function runReplayTestAttempt( platform, target, artifactsDir, + shard, runReplay, cleanupSession, } = params; @@ -64,6 +69,7 @@ export async function runReplayTestAttempt( artifactsDir, artifactPaths, tracePath, + shard, }) .catch((error) => { const appErr = normalizeError(error); diff --git a/src/daemon/handlers/session-test-sharding.ts b/src/daemon/handlers/session-test-sharding.ts new file mode 100644 index 000000000..b8a97c345 --- /dev/null +++ b/src/daemon/handlers/session-test-sharding.ts @@ -0,0 +1,198 @@ +import { listDeviceInventory, type DeviceInventoryRequest } from '../../core/dispatch-resolve.ts'; +import { + resolveAndroidSerialAllowlist, + resolveIosSimulatorDeviceSetPath, +} from '../../utils/device-isolation.ts'; +import { + matchesPlatformSelector, + resolveAppleSimulatorSetPathForSelector, + type DeviceInfo, +} from '../../utils/device.ts'; +import { AppError } from '../../utils/errors.ts'; +import type { CommandFlags } from '../../core/dispatch.ts'; + +export type ReplayTestShardMode = 'all' | 'split'; + +export type ReplayTestShardContext = { + shardIndex: number; + shardCount: number; + device: DeviceInfo; +}; + +export type ReplayTestShardPlan = { + mode: ReplayTestShardMode; + shardCount: number; + total: number; + shards: Array; +}; + +export async function buildReplayTestShardPlan( + flags: CommandFlags | undefined, + runnableEntries: TEntry[], + skippedCount: number, +): Promise | undefined> { + const mode = readReplayTestShardMode(flags); + if (!mode) return undefined; + if (runnableEntries.length === 0) return undefined; + + const devices = await resolveReplayTestShardDevices(flags, mode.count); + return { + mode: mode.kind, + shardCount: mode.count, + total: + skippedCount + + (mode.kind === 'all' ? runnableEntries.length * mode.count : runnableEntries.length), + shards: devices.map((device, index) => ({ + shardIndex: index, + shardCount: mode.count, + device, + entries: + mode.kind === 'all' + ? runnableEntries + : runnableEntries.filter((_entry, entryIndex) => entryIndex % mode.count === index), + })), + }; +} + +export function buildReplayTestShardFlags( + parentFlags: CommandFlags | undefined, + shard: ReplayTestShardContext | undefined, +): CommandFlags | undefined { + if (!shard) return parentFlags; + const base = { + ...(parentFlags ?? {}), + device: undefined, + udid: undefined, + serial: undefined, + platform: shard.device.platform, + target: shard.device.target, + shardAll: undefined, + shardSplit: undefined, + shardCount: shard.shardCount, + shardIndex: shard.shardIndex, + }; + return shard.device.platform === 'android' + ? { ...base, serial: shard.device.id } + : { ...base, udid: shard.device.id }; +} + +function readReplayTestShardMode( + flags: CommandFlags | undefined, +): { kind: ReplayTestShardMode; count: number } | undefined { + const shardAll = readPositiveShardCount(flags?.shardAll, '--shard-all'); + const shardSplit = readPositiveShardCount(flags?.shardSplit, '--shard-split'); + if (shardAll !== undefined && shardSplit !== undefined) { + throw new AppError('INVALID_ARGS', '--shard-all and --shard-split are mutually exclusive'); + } + if (shardAll !== undefined) return { kind: 'all', count: shardAll }; + if (shardSplit !== undefined) return { kind: 'split', count: shardSplit }; + return undefined; +} + +function readPositiveShardCount(value: unknown, flagName: string): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) { + throw new AppError('INVALID_ARGS', `${flagName} requires a positive integer`); + } + return value; +} + +async function resolveReplayTestShardDevices( + flags: CommandFlags | undefined, + shardCount: number, +): Promise { + const explicitSelectors = explicitShardDeviceSelectors(flags); + const inventory = await listDeviceInventory(buildReplayTestShardInventoryRequest(flags)); + const devices = selectReplayTestShardDevices(inventory, explicitSelectors, flags); + + if (devices.length < shardCount) { + throw new AppError( + 'DEVICE_NOT_FOUND', + `test sharding requires ${formatDeviceCount(shardCount)}, but only ${devices.length} matched`, + ); + } + return devices.slice(0, shardCount); +} + +function buildReplayTestShardInventoryRequest( + flags: CommandFlags | undefined, +): DeviceInventoryRequest { + const androidSerialAllowlist = resolveAndroidSerialAllowlist(flags?.androidDeviceAllowlist); + return { + platform: flags?.platform, + target: flags?.target, + iosSimulatorSetPath: resolveAppleSimulatorSetPathForSelector({ + simulatorSetPath: resolveIosSimulatorDeviceSetPath(flags?.iosSimulatorDeviceSet), + platform: flags?.platform, + target: flags?.target, + }), + androidSerialAllowlist: androidSerialAllowlist + ? Array.from(androidSerialAllowlist).sort() + : undefined, + }; +} + +function selectReplayTestShardDevices( + inventory: DeviceInfo[], + explicitSelectors: string[], + flags: CommandFlags | undefined, +): DeviceInfo[] { + if (explicitSelectors.length > 0) { + return resolveExplicitShardDevices(inventory, explicitSelectors, flags); + } + return inventory + .filter((device) => isImplicitShardDevice(device, flags)) + .sort(compareShardDevices); +} + +function formatDeviceCount(count: number): string { + return `${count} device${count === 1 ? '' : 's'}`; +} + +function explicitShardDeviceSelectors(flags: CommandFlags | undefined): string[] { + const raw = flags?.device; + if (typeof raw !== 'string' || raw.trim().length === 0) return []; + return raw + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function resolveExplicitShardDevices( + inventory: DeviceInfo[], + selectors: string[], + flags: CommandFlags | undefined, +): DeviceInfo[] { + return selectors.map((selector) => { + const normalizedSelector = normalizeDeviceName(selector); + const match = inventory.find( + (device) => + isShardDeviceCandidate(device, flags) && + (device.id === selector || normalizeDeviceName(device.name) === normalizedSelector), + ); + if (!match) { + throw new AppError('DEVICE_NOT_FOUND', `No shard device matched ${selector}`); + } + return match; + }); +} + +function isImplicitShardDevice(device: DeviceInfo, flags: CommandFlags | undefined): boolean { + if (!isShardDeviceCandidate(device, flags)) return false; + if (device.platform !== 'ios' && device.platform !== 'android') return false; + return device.booted !== false; +} + +function isShardDeviceCandidate(device: DeviceInfo, flags: CommandFlags | undefined): boolean { + if (!matchesPlatformSelector(device.platform, flags?.platform)) return false; + if (flags?.target && (device.target ?? 'mobile') !== flags.target) return false; + return true; +} + +function normalizeDeviceName(value: string): string { + return value.toLowerCase().replace(/_/g, ' ').replace(/\s+/g, ' ').trim(); +} + +function compareShardDevices(a: DeviceInfo, b: DeviceInfo): number { + return a.id.localeCompare(b.id); +} diff --git a/src/daemon/handlers/session-test-types.ts b/src/daemon/handlers/session-test-types.ts index 669fa8d6e..a64f72131 100644 --- a/src/daemon/handlers/session-test-types.ts +++ b/src/daemon/handlers/session-test-types.ts @@ -1,5 +1,6 @@ import type { DaemonResponse } from '../types.ts'; import type { ReplayScriptMetadata } from '../../replay/script.ts'; +import type { ReplayTestShardContext } from './session-test-sharding.ts'; export type ReplayTestRunReplayParams = { filePath: string; @@ -10,6 +11,7 @@ export type ReplayTestRunReplayParams = { artifactsDir?: string; artifactPaths?: Set; tracePath?: string; + shard?: ReplayTestShardContext; }; export type ReplayTestRunReplay = (params: ReplayTestRunReplayParams) => Promise; diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 7dd3dcf31..3e0c9ce0d 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import { asAppError } from '../../utils/errors.ts'; +import { asAppError, normalizeError } from '../../utils/errors.ts'; import { errorResponse } from './response.ts'; import type { DaemonRequest, @@ -26,6 +26,10 @@ import { import { isReplayInfrastructureFailure } from './session-test-infrastructure.ts'; import { runReplayTestAttempt } from './session-test-runtime.ts'; import type { ReplayTestRuntimeDependencies } from './session-test-types.ts'; +import { buildReplayTestShardPlan, type ReplayTestShardContext } from './session-test-sharding.ts'; + +type ReplayTestEntry = ReturnType[number]; +type ReplayTestRunEntry = Extract; // fallow-ignore-next-line complexity export async function runReplayTestSuite( @@ -54,51 +58,54 @@ export async function runReplayTestSuite( suiteInvocationId, }); - const results: ReplaySuiteTestResult[] = []; const suiteStartedAt = Date.now(); - let executed = 0; - - for (const [entryIndex, entry] of entries.entries()) { - if (entry.kind === 'skip') { - emitRequestProgress({ - type: 'replay-test', - file: entry.path, - status: 'skip', - index: entryIndex + 1, - total: entries.length, - message: entry.message, - }); - results.push({ - file: entry.path, - status: 'skipped', - durationMs: 0, - reason: entry.reason, - message: entry.message, - }); - continue; - } + const skipped = entries.filter((entry) => entry.kind === 'skip'); + const runnable = entries.filter((entry): entry is ReplayTestRunEntry => entry.kind === 'run'); + const shardPlan = await buildReplayTestShardPlan(req.flags, runnable, skipped.length); + const results: ReplaySuiteTestResult[] = shardPlan + ? emitSkippedReplayTestResults({ + entries, + total: shardPlan.total, + }) + : []; - executed += 1; - const result = await runReplayTestCase({ - entry, - sessionName, - suiteInvocationId, - caseIndex: executed - 1, - cwd: req.meta?.cwd, - requestId: req.meta?.requestId, - retries: resolveReplayTestRetries(req.flags?.retries, entry.metadata.retries), - timeoutMs: resolveReplayTestTimeout(req.flags?.timeoutMs, entry.metadata.timeoutMs), - suiteArtifactsDir, - suiteIndex: entryIndex + 1, - suiteTotal: entries.length, - runReplay, - cleanupSession, - }); - results.push(result); - if (req.flags?.failFast === true || isReplayInfrastructureFailure(result)) break; + if (shardPlan) { + results.push( + ...(await runReplayTestShards({ + shards: shardPlan.shards, + sessionName, + suiteInvocationId, + cwd: req.meta?.cwd, + requestId: req.meta?.requestId, + flags: req.flags, + suiteArtifactsDir, + suiteTotal: shardPlan.total, + runReplay, + cleanupSession, + })), + ); + } else { + results.push( + ...(await runReplayTestEntriesInDiscoveryOrder({ + discoveryEntries: entries, + sessionName, + suiteInvocationId, + cwd: req.meta?.cwd, + requestId: req.meta?.requestId, + flags: req.flags, + suiteArtifactsDir, + suiteTotal: entries.length, + runReplay, + cleanupSession, + })), + ); } - const data = summarizeReplayTestResults(entries.length, results, Date.now() - suiteStartedAt); + const data = summarizeReplayTestResults( + shardPlan?.total ?? entries.length, + results, + Date.now() - suiteStartedAt, + ); return { ok: true, data }; } catch (err) { const appErr = asAppError(err); @@ -106,15 +113,230 @@ export async function runReplayTestSuite( } } +function emitSkippedReplayTestResults(params: { + entries: ReplayTestEntry[]; + total: number; +}): ReplaySuiteTestResult[] { + const { entries, total } = params; + const results: ReplaySuiteTestResult[] = []; + for (const [entryIndex, entry] of entries.entries()) { + if (entry.kind !== 'skip') continue; + emitRequestProgress({ + type: 'replay-test', + file: entry.path, + status: 'skip', + index: entryIndex + 1, + total, + message: entry.message, + }); + results.push({ + file: entry.path, + status: 'skipped', + durationMs: 0, + reason: entry.reason, + message: entry.message, + }); + } + return results; +} + +async function runReplayTestShards( + params: { + shards: Array; + sessionName: string; + suiteInvocationId: string; + cwd?: string; + requestId?: string; + flags: DaemonRequest['flags']; + suiteArtifactsDir: string; + suiteTotal: number; + } & ReplayTestRuntimeDependencies, +): Promise { + const settled = await Promise.allSettled( + params.shards.map(async (shard) => await runReplayTestShard({ ...params, shard })), + ); + return settled.flatMap((result, index) => { + if (result.status === 'fulfilled') return result.value; + const shard = params.shards[index]; + return shard ? [buildUnexpectedShardFailure(shard, params.sessionName, result.reason)] : []; + }); +} + +function buildUnexpectedShardFailure( + shard: ReplayTestShardContext & { entries: ReplayTestRunEntry[] }, + sessionName: string, + reason: unknown, +): ReplaySuiteTestFailed { + const appErr = normalizeError(reason); + return { + file: shard.entries[0]?.path ?? `shard-${shard.shardIndex + 1}`, + session: formatReplayTestShardSessionName(sessionName, shard), + status: 'failed', + durationMs: 0, + attempts: 1, + error: { + code: appErr.code, + message: appErr.message, + hint: appErr.hint, + diagnosticId: appErr.diagnosticId, + logPath: appErr.logPath, + details: appErr.details, + }, + shardIndex: shard.shardIndex, + shardCount: shard.shardCount, + deviceId: shard.device.id, + }; +} + +async function runReplayTestShard( + params: { + shard: ReplayTestShardContext & { entries: ReplayTestRunEntry[] }; + sessionName: string; + suiteInvocationId: string; + cwd?: string; + requestId?: string; + flags: DaemonRequest['flags']; + suiteArtifactsDir: string; + suiteTotal: number; + } & ReplayTestRuntimeDependencies, +): Promise { + const { shard, sessionName } = params; + return await runReplayTestEntries({ + ...params, + entries: shard.entries, + sessionName: formatReplayTestShardSessionName(sessionName, shard), + shard, + }); +} + +function formatReplayTestShardSessionName( + sessionName: string, + shard: ReplayTestShardContext, +): string { + return `${sessionName}:shard-${shard.shardIndex + 1}`; +} + +async function runReplayTestEntriesInDiscoveryOrder( + params: { + discoveryEntries: ReplayTestEntry[]; + sessionName: string; + suiteInvocationId: string; + cwd?: string; + requestId?: string; + flags: DaemonRequest['flags']; + suiteArtifactsDir: string; + suiteTotal: number; + } & ReplayTestRuntimeDependencies, +): Promise { + const { + discoveryEntries, + sessionName, + suiteInvocationId, + cwd, + requestId, + flags, + suiteArtifactsDir, + suiteTotal, + runReplay, + cleanupSession, + } = params; + const results: ReplaySuiteTestResult[] = []; + let executed = 0; + for (const [entryIndex, entry] of discoveryEntries.entries()) { + if (entry.kind === 'skip') { + emitRequestProgress({ + type: 'replay-test', + file: entry.path, + status: 'skip', + index: entryIndex + 1, + total: suiteTotal, + message: entry.message, + }); + results.push({ + file: entry.path, + status: 'skipped', + durationMs: 0, + reason: entry.reason, + message: entry.message, + }); + continue; + } + executed += 1; + const result = await runReplayTestCase({ + entry, + sessionName, + suiteInvocationId, + caseIndex: executed - 1, + cwd, + requestId, + retries: resolveReplayTestRetries(flags?.retries, entry.metadata.retries), + timeoutMs: resolveReplayTestTimeout(flags?.timeoutMs, entry.metadata.timeoutMs), + suiteArtifactsDir, + suiteIndex: entryIndex + 1, + suiteTotal, + runReplay, + cleanupSession, + }); + results.push(result); + if (flags?.failFast === true || isReplayInfrastructureFailure(result)) break; + } + return results; +} + +async function runReplayTestEntries( + params: { + entries: ReplayTestRunEntry[]; + sessionName: string; + suiteInvocationId: string; + cwd?: string; + requestId?: string; + flags: DaemonRequest['flags']; + suiteArtifactsDir: string; + suiteTotal: number; + shard?: ReplayTestShardContext; + } & ReplayTestRuntimeDependencies, +): Promise { + const { + entries, + sessionName, + suiteInvocationId, + cwd, + requestId, + flags, + suiteArtifactsDir, + suiteTotal, + shard, + runReplay, + cleanupSession, + } = params; + const results: ReplaySuiteTestResult[] = []; + for (const [entryIndex, entry] of entries.entries()) { + const result = await runReplayTestCase({ + entry, + sessionName, + suiteInvocationId, + caseIndex: entryIndex, + cwd, + requestId, + retries: resolveReplayTestRetries(flags?.retries, entry.metadata.retries), + timeoutMs: resolveReplayTestTimeout(flags?.timeoutMs, entry.metadata.timeoutMs), + suiteArtifactsDir, + suiteIndex: entryIndex + 1, + suiteTotal, + shard, + runReplay, + cleanupSession, + }); + results.push(result); + if (flags?.failFast === true || isReplayInfrastructureFailure(result)) break; + } + return results; +} + // fallow-ignore-next-line complexity async function runReplayTestCase( params: { - entry: Extract< - ReturnType[number], - { - kind: 'run'; - } - >; + entry: ReplayTestRunEntry; sessionName: string; suiteInvocationId: string; caseIndex: number; @@ -125,6 +347,7 @@ async function runReplayTestCase( suiteArtifactsDir: string; suiteIndex: number; suiteTotal: number; + shard?: ReplayTestShardContext; } & ReplayTestRuntimeDependencies, ): Promise> { const { @@ -139,12 +362,14 @@ async function runReplayTestCase( suiteArtifactsDir, suiteIndex, suiteTotal, + shard, runReplay, cleanupSession, } = params; const testStartedAt = Date.now(); const testArtifactsDir = path.join( suiteArtifactsDir, + ...(shard ? [`shard-${shard.shardIndex + 1}`] : []), buildReplayTestArtifactSlug(entry.path, cwd), ); let finalResponse: DaemonResponse | undefined; @@ -174,6 +399,7 @@ async function runReplayTestCase( filePath: entry.path, caseIndex, attemptIndex, + shardIndex: shard?.shardIndex, }); const response = await runReplayTestAttempt({ filePath: entry.path, @@ -183,6 +409,7 @@ async function runReplayTestCase( platform: entry.metadata.platform, target: entry.metadata.target, artifactsDir: attemptArtifactsDir, + shard, runReplay, cleanupSession, }); @@ -245,6 +472,7 @@ async function runReplayTestCase( artifactsDir: testArtifactsDir, replayed: typeof finalResponse.data?.replayed === 'number' ? finalResponse.data.replayed : 0, healed: typeof finalResponse.data?.healed === 'number' ? finalResponse.data.healed : 0, + ...replayTestShardResultMetadata(shard), ...(attemptFailures.length > 0 ? { attemptFailures } : {}), }; } @@ -277,9 +505,22 @@ async function runReplayTestCase( attempts, artifactsDir: testArtifactsDir, error, + ...replayTestShardResultMetadata(shard), }; } +function replayTestShardResultMetadata( + shard: ReplayTestShardContext | undefined, +): Pick { + return shard + ? { + shardIndex: shard.shardIndex, + shardCount: shard.shardCount, + deviceId: shard.device.id, + } + : {}; +} + function summarizeReplayTestResults( total: number, results: ReplaySuiteTestResult[], diff --git a/src/daemon/request-platform-providers.ts b/src/daemon/request-platform-providers.ts index 5f9209726..db8685f0e 100644 --- a/src/daemon/request-platform-providers.ts +++ b/src/daemon/request-platform-providers.ts @@ -280,17 +280,36 @@ async function resolveScopedProviderDevice( existingSession: SessionState | undefined, ): Promise { if (existingSession) { - return shouldPreferExplicitDeviceOverExistingSession(req) && - hasExplicitDeviceSelector(req.flags) - ? await resolveTargetDevice(req.flags ?? {}) - : existingSession.device; - } - if (!hasExplicitDeviceSelector(req.flags) && !usesSessionlessDefaultProviderDevice(req)) { - return undefined; + return await resolveExistingSessionProviderDevice(req, existingSession); } + if (shouldSkipSessionlessProviderDevice(req)) return undefined; + return await resolveTargetDevice(req.flags ?? {}); +} + +async function resolveExistingSessionProviderDevice( + req: DaemonRequest, + existingSession: SessionState, +): Promise { + if (!shouldResolveExplicitProviderDevice(req)) return existingSession.device; return await resolveTargetDevice(req.flags ?? {}); } +function shouldResolveExplicitProviderDevice(req: DaemonRequest): boolean { + return shouldPreferExplicitDeviceOverExistingSession(req) && hasExplicitDeviceSelector(req.flags); +} + +function shouldSkipSessionlessProviderDevice(req: DaemonRequest): boolean { + if (isShardedReplayTestRequest(req)) return true; + return !hasExplicitDeviceSelector(req.flags) && !usesSessionlessDefaultProviderDevice(req); +} + +function isShardedReplayTestRequest(req: DaemonRequest): boolean { + return ( + req.command === 'test' && + (typeof req.flags?.shardAll === 'number' || typeof req.flags?.shardSplit === 'number') + ); +} + async function requestPlatformProviderScopeWrappers( scopedProviders: ResolvedRequestPlatformProviders, ): Promise { diff --git a/src/daemon/types.ts b/src/daemon/types.ts index 302d0d620..c497ca734 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -48,6 +48,9 @@ export type ReplaySuiteTestPassed = { replayed: number; healed: number; attemptFailures?: ReplaySuiteAttemptFailure[]; + shardIndex?: number; + shardCount?: number; + deviceId?: string; }; export type ReplaySuiteTestFailed = { @@ -66,6 +69,9 @@ export type ReplaySuiteTestFailed = { logPath?: string; details?: Record; }; + shardIndex?: number; + shardCount?: number; + deviceId?: string; }; export type ReplaySuiteTestSkipped = { diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 572a09ef5..e01dcdb69 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -1033,6 +1033,9 @@ test('usageForCommand includes Maestro test suite flag', () => { if (help === null) throw new Error('Expected test help text'); assert.match(help, /Run one or more replay scripts as a serial test suite/); assert.match(help, /--maestro/); + assert.match(help, /--shard-all /); + assert.match(help, /--shard-split /); + assert.match(help, /AD_SHARD_INDEX is zero-based/); assert.match(help, /Replay\/Test: inject or override/); }); @@ -1488,6 +1491,7 @@ test('command usage describes test suite flags', () => { assert.match(help, /Run one or more replay scripts as a serial test suite/); assert.match(help, /--maestro/); assert.match(help, /--fail-fast/); + assert.match(help, /each shard stops independently/); assert.match(help, /--timeout /); assert.match(help, /--retries /); assert.match(help, /--artifacts-dir /); diff --git a/src/utils/cli-command-overrides.ts b/src/utils/cli-command-overrides.ts index 163f24d6a..470baff1f 100644 --- a/src/utils/cli-command-overrides.ts +++ b/src/utils/cli-command-overrides.ts @@ -233,6 +233,8 @@ const CLI_COMMAND_OVERRIDES = { 'retries', 'artifactsDir', 'reportJunit', + 'shardAll', + 'shardSplit', ], }, batch: { diff --git a/src/utils/cli-flags.ts b/src/utils/cli-flags.ts index 3a42b706c..05c9ca7f6 100644 --- a/src/utils/cli-flags.ts +++ b/src/utils/cli-flags.ts @@ -92,6 +92,8 @@ export type CliFlags = RemoteConfigMetroOptions & retries?: number; artifactsDir?: string; reportJunit?: string; + shardAll?: number; + shardSplit?: number; steps?: string; stepsFile?: string; findFirst?: boolean; @@ -792,7 +794,8 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ names: ['--fail-fast'], type: 'boolean', usageLabel: '--fail-fast', - usageDescription: 'Test: stop the suite after the first failing script', + usageDescription: + 'Test: stop the suite after the first failing script; with sharding, each shard stops independently', }, { key: 'timeoutMs', @@ -800,7 +803,8 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'int', min: 1, usageLabel: '--timeout ', - usageDescription: 'Prepare/Replay/Snapshot/Test: maximum wall-clock time for the command or attempt', + usageDescription: + 'Prepare/Replay/Snapshot/Test: maximum wall-clock time for the command or attempt', }, { key: 'retries', @@ -825,6 +829,23 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--report-junit ', usageDescription: 'Test: write a JUnit XML report for the replay suite', }, + { + key: 'shardAll', + names: ['--shard-all'], + type: 'int', + min: 1, + usageLabel: '--shard-all ', + usageDescription: 'Test: run the full suite on each of n devices; AD_SHARD_INDEX is zero-based', + }, + { + key: 'shardSplit', + names: ['--shard-split'], + type: 'int', + min: 1, + usageLabel: '--shard-split ', + usageDescription: + 'Test: split runnable suite entries across n devices; AD_SHARD_INDEX is zero-based', + }, { key: 'steps', names: ['--steps'], diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 837bbc360..6979830b2 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1807,6 +1807,24 @@ const SKILL_GUIDANCE_CASES: Case[] = [ ], forbiddenOutputs: [/maestro\s+test/i, /maestro\s+cloud/i, plannedCommand('test')], }), + makeCase({ + id: 'test-maestro-shard-all-devices', + contract: [ + 'Suite path: ./e2e/maestro', + 'The suite contains Maestro YAML compatibility flows', + 'Connected device ids: udid1, emulator-5554', + 'Need local cross-device validation by running the full suite on each device', + ], + task: 'Plan the Agent Device test command that runs the Maestro suite on both connected devices without calling the Maestro CLI directly.', + outputs: [ + plannedCommand('test'), + /--maestro/i, + /--device\s+["']?udid1,emulator-5554["']?/i, + /--shard-all\s+2/i, + /\.\/e2e\/maestro/i, + ], + forbiddenOutputs: [/maestro\s+test/i, /--shard-split/i, /--platform\s+(?:ios|android)/i], + }), makeCase({ id: 'batch-known-stable-flow', contract: [