test: parallelize the integration pool and stop rebuilding shared artifacts (reland)

examples/rsc-agent-runtime/src/dev/rsbuild-runtime-session.ts

@@ -93,7 +93,8 @@ const maximumInvocationWorkers = 4;
 const maximumInvocationStdoutBytes = 4 * 1024 * 1024;
 const maximumInvocationFlightBytes = 4 * 1024 * 1024;
 const maximumInvocationStderrBytes = 256 * 1024;
-const maximumRunHistory = 50;
+/** Production terminal-run retention window; tests may shrink it through the start testing seam. */
+export const defaultMaximumRunHistory = 50;
 const invocationTimeoutMs = 10_000;
 const invocationTerminationGraceMs = 100;
 const flightPreviewBytes = 32 * 1024;
@@ -603,6 +604,12 @@ export interface RsbuildRuntimeSessionStartTesting {
   }>) => Promise<void> | void;
   /** Windows-only Job owner fault injection; never used by the public provider. */
   readonly windowsJobOwnerMode?: 'close-control' | 'hang-ready' | 'ignore-stop' | 'nonzero-after-drain' | 'normal';
+  /**
+   * Test-only terminal-run retention override so eviction suites do not need
+   * fifty real invocations; the public provider always keeps
+   * `defaultMaximumRunHistory` runs.
+   */
+  readonly maximumRunHistory?: number;
 }
 
 /**
@@ -633,6 +640,7 @@ export class RsbuildRuntimeSession implements DevRuntimeSession {
   readonly #surfaceAssetApps = new Map<string, DevRuntimePreparedProject['apps'][number]>();
   readonly #surfaces = new Map<string, DevRuntimeSurface>();
   readonly #testing: RsbuildRuntimeSessionStartTesting;
+  readonly #maximumRunHistory: number;
   readonly #attempts = new Map<string, AttemptBarrier>();
   readonly #workers = new Map<string, InvocationWorker>();
   readonly #failedAttempts = new Set<string>();
@@ -669,6 +677,7 @@ export class RsbuildRuntimeSession implements DevRuntimeSession {
     this.#mcpRegistry = input.mcpRegistry;
     this.#latestPreparedRuntime = input.preparedRuntime;
     this.#testing = input.testing;
+    this.#maximumRunHistory = input.testing.maximumRunHistory ?? defaultMaximumRunHistory;
     this.#ownedRunsRoot = input.ownedRunsRoot;
     this.#runRoot = input.ownedRunsRoot.root;
     this.#stateFile = join(resolve(input.context.storageRoot), 'state', `${stateStoreId}.jsonl`);
@@ -1035,8 +1044,8 @@ export class RsbuildRuntimeSession implements DevRuntimeSession {
 
   runs(limit: number): readonly DevRuntimeRun[] {
     if (this.#closed) return Object.freeze([]);
-    if (!Number.isSafeInteger(limit) || limit < 1 || limit > maximumRunHistory) {
-      throw new RangeError(`Runtime run history limit must be an integer from 1 through ${maximumRunHistory}.`);
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > this.#maximumRunHistory) {
+      throw new RangeError(`Runtime run history limit must be an integer from 1 through ${String(this.#maximumRunHistory)}.`);
     }
     return Object.freeze([...this.#terminalRuns.values()].reverse().slice(0, limit));
   }
@@ -1333,7 +1342,7 @@ export class RsbuildRuntimeSession implements DevRuntimeSession {
   }
 
   async #evictTerminalRuns(): Promise<void> {
-    while (this.#terminalRuns.size > maximumRunHistory) {
+    while (this.#terminalRuns.size > this.#maximumRunHistory) {
       const oldestId = this.#terminalRuns.keys().next().value as string | undefined;
       if (oldestId === undefined) return;
       this.#evictingTerminalRuns.add(oldestId);

package.json

@@ -12,8 +12,8 @@
     "lint:package": "publint packages/agent-bundle",
     "test": "pnpm test:unit && pnpm test:integration",
     "test:unit": "rstest --config rstest.unit.config.ts",
-    "test:integration": "pnpm --filter agent-bundle-workbench build && pnpm test:integration:run",
-    "test:integration:run": "AGENT_BUNDLE_WORKBENCH_PREBUILT=1 rstest --config rstest.integration.config.ts --pool.maxWorkers 1",
+    "test:integration": "pnpm build && pnpm test:integration:run",
+    "test:integration:run": "AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.integration.config.ts && AGENT_BUNDLE_WORKBENCH_PREBUILT=1 AGENT_BUNDLE_PACKAGE_PREBUILT=1 rstest --config rstest.integration-serial.config.ts",
     "test:watch": "rstest --config rstest.config.ts --watch",
     "lint": "rslint .",
     "typecheck": "tsc --noEmit && tsc --project packages/workbench/tsconfig.json",

packages/agent-bundle/tests/cli.test.ts

@@ -15,6 +15,7 @@ const cliPath = join(packageRoot, 'dist/cli.js');
 let buildPackage: Promise<void> | undefined;
 
 const buildCliPackage = async (): Promise<void> => {
+  if (process.env['AGENT_BUNDLE_PACKAGE_PREBUILT'] === '1') return;
   buildPackage ??= execFile('pnpm', ['build'], { cwd: workspaceRoot }).then(() => undefined);
   await buildPackage;
 };

packages/agent-bundle/tests/support/time-scale.ts

@@ -4,5 +4,12 @@
  * processes, and rsbuild compiles inside a single test. Scaling the budgets
  * costs nothing on green runs - polling assertions return on success - and
  * the workflow-level timeout-minutes still bounds real hangs.
+ *
+ * AGENT_BUNDLE_TEST_TIME_SCALE (set by rstest.integration.config.ts when the
+ * pool runs multiple workers) covers the same contention on development
+ * machines, where concurrent Chrome + dev-server + rsbuild pairs share cores.
  */
-export const timeScale = process.env['CI'] === undefined ? 1 : 4;
+const localScale = Number(process.env['AGENT_BUNDLE_TEST_TIME_SCALE'] ?? '');
+export const timeScale = process.env['CI'] !== undefined
+  ? 4
+  : Number.isSafeInteger(localScale) && localScale >= 1 ? localScale : 1;

packages/workbench/tests/evals-real.e2e.test.ts

@@ -14,11 +14,12 @@ import { createProjectFixture, removeProjectFixture } from '../../agent-bundle/t
 import { seedEvalProject, writeEvalSuite } from '../../agent-bundle/tests/support/eval-project.ts';
 import { readFinalizedEvalRun } from '../src/evals/evals-page.tsx';
 import { closeServer } from './support/http.ts';
+import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
 import { workbenchBrowserAliases } from './support/workbench-browser-modules.ts';
 import { buildWorkbench, e2e, workbenchAssets, workspaceRoot, workbenchUrl } from './support/workbench-e2e.ts';
 
 const evalsPage = join(workspaceRoot, 'packages', 'workbench', 'src', 'evals', 'evals-page.tsx');
-const browserTimeout = 12_000;
+const browserTimeout = 12_000 * timeScale;
 const runCompletionTimeout = 60_000;
 
 e2e('retries a terminal canonical read until the durable run finalization is visible', async () => {

packages/workbench/tests/examples-real.e2e.test.ts

@@ -14,9 +14,10 @@ import {
   waitForSettledWorkbench,
   writeExampleReport,
 } from './support/example-acceptance.ts';
+import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
 import { buildWorkbench, e2e, workbenchAssets, workbenchUrl } from './support/workbench-e2e.ts';
 
-const browserTimeout = 15_000;
+const browserTimeout = 15_000 * timeScale;
 
 const waitForExampleValue = async <Value>(
   page: Parameters<typeof captureExampleState>[0],

packages/workbench/tests/logs-real.e2e.test.ts

@@ -5,9 +5,10 @@ import { expect } from '@rstest/playwright';
 import { createWorkbenchAssetSource } from '../../agent-bundle/src/dev/workbench-assets.ts';
 import { startDevServer } from '../../agent-bundle/src/dev/workbench-server.ts';
 import { createProjectFixture, removeProjectFixture } from '../../agent-bundle/tests/helpers/project-fixture.ts';
+import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
 import { buildWorkbench, e2e, workbenchAssets, workbenchUrl } from './support/workbench-e2e.ts';
 
-const browserTimeout = 12_000;
+const browserTimeout = 12_000 * timeScale;
 
 e2e('shows real producer logs with replay, filters, redaction, responsive layout, and no browser errors', { timeout: 90_000 }, async ({ page }) => {
   await buildWorkbench();

packages/workbench/tests/mcp-app-real.e2e.test.ts

@@ -1,7 +1,5 @@
-import { execFile as executeFile } from 'node:child_process';
 import { access, mkdir, readFile, symlink, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { promisify } from 'node:util';
 
 import { expect, test, type PlaywrightOptions } from '@rstest/playwright';
 import type { Page, WebSocketRoute } from 'playwright';
@@ -12,12 +10,11 @@ import { startDevServer } from '../../agent-bundle/src/dev/workbench-server.ts';
 import { createProjectFixture, removeProjectFixture } from '../../agent-bundle/tests/helpers/project-fixture.ts';
 import { startRuntimePlaygroundFixture } from './helpers/runtime-playground-fixture.ts';
 import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
-import { workbenchUrl } from './support/workbench-e2e.ts';
+import { buildWorkbench, workbenchUrl } from './support/workbench-e2e.ts';
 
 const workspaceRoot = process.cwd();
 const workbenchAssets = join(workspaceRoot, 'packages', 'workbench', 'dist');
 const browserTimeout = 8_000 * timeScale;
-const execFile = promisify(executeFile);
 
 const e2e = test.extend({
   playwright: {
@@ -26,14 +23,6 @@ const e2e = test.extend({
   } satisfies PlaywrightOptions,
 });
 
-const buildWorkbench = async (): Promise<void> => {
-  const { RSTEST: _rstest, ...environment } = process.env;
-  await execFile('pnpm', ['--filter', 'agent-bundle-workbench', 'build'], {
-    cwd: workspaceRoot,
-    env: { ...environment, NODE_ENV: 'production' },
-  });
-};
-
 const appFixtureHtml = [
   '<!doctype html><html><body><main data-testid="app-state">waiting</main>',
   '<script>',

packages/workbench/tests/overview.e2e.test.ts

@@ -1,7 +1,5 @@
-import { execFile as executeFile } from 'node:child_process';
 import { mkdir, readFile, rename, symlink, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { promisify } from 'node:util';
 
 import { expect, test, type PlaywrightOptions } from '@rstest/playwright';
 import type { Locator, Page } from 'playwright';
@@ -21,8 +19,8 @@ import { startDevServer } from '../../agent-bundle/src/dev/workbench-server.ts';
 import { createProjectFixture, removeProjectFixture } from '../../agent-bundle/tests/helpers/project-fixture.ts';
 import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
 import { startRuntimePlaygroundFixture } from './helpers/runtime-playground-fixture.ts';
+import { buildWorkbench } from './support/workbench-e2e.ts';
 
-const execFile = promisify(executeFile);
 const workspaceRoot = process.cwd();
 const workbenchAssets = join(workspaceRoot, 'packages', 'workbench', 'dist');
 const browserTimeout = 15_000 * timeScale;
@@ -55,14 +53,6 @@ const e2e = test.extend({
   } satisfies PlaywrightOptions,
 });
 
-const buildWorkbench = async (): Promise<void> => {
-  const { RSTEST: _rstest, ...environment } = process.env;
-  await execFile('pnpm', ['--filter', 'agent-bundle-workbench', 'build'], {
-    cwd: workspaceRoot,
-    env: { ...environment, NODE_ENV: 'production' },
-  });
-};
-
 const startFrozenEpochServer = async (root: string) => {
   const registry = createDefaultRegistry();
   const epochStore = new EpochStore({ projectRoot: root });

packages/workbench/tests/playground-real.e2e.test.ts

@@ -1,20 +1,18 @@
-import { execFile as executeFile } from 'node:child_process';
 import { chmod, mkdir, symlink, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
-import { promisify } from 'node:util';
 
 import { expect, test, type PlaywrightOptions } from '@rstest/playwright';
 
 import { agentBundleNodeModules, workbenchNodeModules } from '../../agent-bundle/tests/helpers/workspace-paths.ts';
 import { createWorkbenchAssetSource } from '../../agent-bundle/src/dev/workbench-assets.ts';
 import { startDevServer } from '../../agent-bundle/src/dev/workbench-server.ts';
 import { createProjectFixture, removeProjectFixture } from '../../agent-bundle/tests/helpers/project-fixture.ts';
-import { workbenchUrl } from './support/workbench-e2e.ts';
+import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
+import { buildWorkbench, workbenchUrl } from './support/workbench-e2e.ts';
 
-const execFile = promisify(executeFile);
 const workspaceRoot = process.cwd();
 const workbenchAssets = join(workspaceRoot, 'packages', 'workbench', 'dist');
-const browserTimeout = 8_000;
+const browserTimeout = 8_000 * timeScale;
 const nativePathFallback = `${dirname(process.execPath)}:/usr/bin:/bin`;
 
 const e2e = test.extend({
@@ -24,16 +22,6 @@ const e2e = test.extend({
   } satisfies PlaywrightOptions,
 });
 
-let workbenchBuild: Promise<void> | undefined;
-
-const buildWorkbench = (): Promise<void> => workbenchBuild ??= (async (): Promise<void> => {
-  const { RSTEST: _rstest, ...environment } = process.env;
-  await execFile('pnpm', ['--filter', 'agent-bundle-workbench', 'build'], {
-    cwd: workspaceRoot,
-    env: { ...environment, NODE_ENV: 'production' },
-  });
-})();
-
 const writeFakeClaude = async (directory: string): Promise<void> => {
   const executable = join(directory, 'claude');
   const implementation = join(directory, 'claude.mjs');

packages/workbench/tests/runtime-playground.e2e.test.ts

@@ -1,9 +1,10 @@
 import { expect, test, type PlaywrightOptions } from '@rstest/playwright';
 
 import { startRuntimePlaygroundFixture } from './helpers/runtime-playground-fixture.ts';
+import { timeScale } from '../../agent-bundle/tests/support/time-scale.ts';
 import { workbenchUrl } from './support/workbench-e2e.ts';
 
-const browserTimeout = 12_000;
+const browserTimeout = 12_000 * timeScale;
 
 const e2e = test.extend({
   playwright: {

packages/workbench/tests/support/packed-release-harness.ts

@@ -31,6 +31,7 @@ export const availablePort = async (): Promise<number> => {
 };
 
 export const buildPackage = (): Promise<void> => builtPackage ??= (async (): Promise<void> => {
+  if (process.env['AGENT_BUNDLE_PACKAGE_PREBUILT'] === '1') return;
   const { RSTEST: _rstest, ...environment } = process.env;
   await execFile('pnpm', ['build'], {
     cwd: workspaceRoot,

rstest.integration-serial.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from '@rstest/core';
+
+import { serialIntegrationTestFiles } from './rstest.integration-tests.ts';
+import { withAgentBundleRslibConfig } from './rstest.rslib.ts';
+
+/**
+ * Integration files that rewrite workspace-shared artifacts (see the
+ * serialIntegrationTestFiles doc in rstest.integration-tests.ts). One worker
+ * only: they rebuild or repack shared package dist directories that every
+ * other file in this group also reads.
+ */
+export default defineConfig({
+  extends: withAgentBundleRslibConfig(),
+  include: [...serialIntegrationTestFiles],
+  pool: { maxWorkers: 1 },
+});

rstest.integration-tests.ts

@@ -69,3 +69,45 @@ export const integrationTestFiles: readonly string[] = [
   'packages/workbench/tests/runtime-playground-hmr.e2e.test.ts',
   'packages/workbench/tests/workbench-dev-command.test.ts',
 ];
+
+/**
+ * Integration files that WRITE to workspace-shared locations and therefore
+ * cannot run alongside other integration files:
+ *
+ * - packed-release.e2e can run a root `pnpm build` (rewriting
+ *   `packages/{agent-bundle,rsc-runtime,workbench}/dist`) when
+ *   AGENT_BUNDLE_PACKAGE_PREBUILT is unset, and always runs `npm pack`
+ *   plus a packed dev server on a pre-reserved (not ephemeral) port.
+ *
+ * They run on one worker via rstest.integration-serial.config.ts after the
+ * parallel pool finishes.
+ */
+export const serialIntegrationTestFiles: readonly string[] = [
+  'packages/workbench/tests/packed-release.e2e.test.ts',
+];
+
+/**
+ * Integration files safe on parallel workers: they create per-test fixtures
+ * with `mkdtemp`, bind servers on ephemeral ports (`port: 0` or rsbuild's
+ * silent free-port fallback), and only READ the prebuilt shared artifacts
+ * (`packages/workbench/dist`, `packages/agent-bundle/dist`).
+ */
+export const parallelIntegrationTestFiles: readonly string[] =
+  integrationTestFiles.filter((file) => !serialIntegrationTestFiles.includes(file));
+
+/**
+ * Pack-and-install tests: each one runs `npm pack` (and usually a clean
+ * `npm install` of the tarball), which dominates the serialized integration
+ * pool. They run through the root `test:packed` / `test:packed:native`
+ * scripts instead — CI's release-gates job (`check:release`) and the
+ * native-host-smoke workflow keep them covered — and stay excluded from the
+ * parallel unit pool.
+ */
+export const packedTestFiles: readonly string[] = [
+  'packages/agent-bundle/tests/dev-workbench-packaging.test.ts',
+  'packages/agent-bundle/tests/packed-consumer.test.ts',
+  'packages/agent-bundle/tests/packed-native-smoke.test.ts',
+  'packages/agent-bundle/tests/public-api-packed.test.ts',
+  'packages/agent-bundle/tests/release-audit.test.ts',
+  'packages/agent-bundle/tests/rsc-runtime-optional-packaging.test.ts',
+];

rstest.integration.config.ts

@@ -1,13 +1,39 @@
+import { availableParallelism } from 'node:os';
+
 import { defineConfig } from '@rstest/core';
 
-import { integrationTestFiles } from './rstest.integration-tests.ts';
+import { parallelIntegrationTestFiles } from './rstest.integration-tests.ts';
 import { withAgentBundleRslibConfig } from './rstest.rslib.ts';
 
-/** Build- and process-running tests: Rslib/Rsbuild caches and output paths are process-shared, so one worker only. */
+/**
+ * Worker count for the parallel integration pool. Half the cores keeps
+ * browser + dev-server pairs from starving each other, the cap of 4 bounds
+ * memory on large machines, and two-core CI still resolves to one worker.
+ * AGENT_BUNDLE_INTEGRATION_MAX_WORKERS overrides the computed value (e.g. to
+ * force a serial run when measuring or bisecting).
+ */
+const overrideWorkers = Number(process.env['AGENT_BUNDLE_INTEGRATION_MAX_WORKERS'] ?? '');
+const maxWorkers = Number.isSafeInteger(overrideWorkers) && overrideWorkers >= 1
+  ? overrideWorkers
+  : Math.max(1, Math.min(4, Math.floor(availableParallelism() / 2)));
+
+/**
+ * Build- and process-running tests that only read workspace-shared artifacts;
+ * files that WRITE shared locations run serialized afterwards through
+ * rstest.integration-serial.config.ts (rstest has no per-project pool or
+ * isolate settings, so the split lives in two configs chained by
+ * `test:integration:run`).
+ */
 export default defineConfig({
   extends: withAgentBundleRslibConfig(),
-  include: [...integrationTestFiles],
-  pool: { maxWorkers: 1 },
+  include: [...parallelIntegrationTestFiles],
+  pool: { maxWorkers },
+  // Concurrent Chrome + dev-server + rsbuild pairs contend for cores, so
+  // parallel runs double the polling budgets (see tests/support/time-scale.ts)
+  // and raise the 5s default test timeout, which real in-process builds can
+  // exceed when workers share the machine. Explicit per-test timeouts win.
+  env: { AGENT_BUNDLE_TEST_TIME_SCALE: maxWorkers > 1 ? '2' : '1' },
+  testTimeout: 30_000,
   // isolate: false would cut Playwright startup cost, but the log pipeline
   // suites rely on per-file module isolation (verified: logs-real.e2e fails
   // when sharing a worker with the other log suites).