From bb50547cfbb1e3355d1e7ba0f3f215396435a7ea Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:53:36 +1000 Subject: [PATCH] test: cut local suite CPU, disk, and E2E matrix Nx was spawning 15 Vitest processes, each with one worker per core. Cap fan-out to 4 tasks and VITEST_MAX_WORKERS=1. Skip empty test:lib targets. Local E2E runs openai/anthropic/gemini and does not record video or retry. CI still runs the full matrix. Closes #1318 --- CONTRIBUTING.md | 20 ++++++++++ nx.json | 2 +- package.json | 8 ++-- packages/ai-react-ui/package.json | 2 - packages/ai-solid-ui/package.json | 2 - packages/ai-vue-ui/package.json | 2 - packages/preact-ai-devtools/package.json | 3 -- packages/preact-ai-devtools/vitest.config.ts | 7 ---- packages/react-ai-devtools/package.json | 3 -- packages/react-ai-devtools/vitest.config.ts | 9 ----- packages/solid-ai-devtools/package.json | 3 -- packages/solid-ai-devtools/vitest.config.ts | 9 ----- packages/svelte-ai-devtools/package.json | 3 -- packages/svelte-ai-devtools/vitest.config.ts | 9 ----- pnpm-lock.yaml | 12 ------ testing/e2e/README.md | 4 +- testing/e2e/playwright.config.ts | 10 +++-- testing/e2e/tests/test-matrix.ts | 40 +++++++++++++++++++- vitest.workspace.js | 16 -------- 19 files changed, 73 insertions(+), 91 deletions(-) delete mode 100644 packages/preact-ai-devtools/vitest.config.ts delete mode 100644 packages/react-ai-devtools/vitest.config.ts delete mode 100644 packages/solid-ai-devtools/vitest.config.ts delete mode 100644 packages/svelte-ai-devtools/vitest.config.ts delete mode 100644 vitest.workspace.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c051e15e07..5aefd95f7f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,26 @@ All commands are run from the repo root. Nx handles affected detection and cachi Working on a single package? `cd packages/` and use its scripts directly (`pnpm test:lib`, `pnpm test:types`, etc.). +### Faster local runs + +Root `pnpm test`, `pnpm test:pr`, and `pnpm test:lib` set `VITEST_MAX_WORKERS=1`. Nx runs 4 tasks at a time (`nx.json` `parallel`). Together that stops each package from spawning one Vitest worker per CPU core while 15 packages already run in parallel. + +A single package still uses all cores: + +```bash +cd packages/ai +pnpm test:lib +``` + +Local `pnpm test:e2e` runs openai, anthropic, and gemini. Features that none of those support (TTS, image-gen, and similar) still run on the providers that do. CI always runs the full matrix. + +```bash +E2E_PROVIDERS=* pnpm test:e2e # full matrix locally +E2E_PROVIDERS=grok pnpm test:e2e # one provider +``` + +Playwright does not retry or record video locally. CI retries twice and keeps the video of a failed test. + ## TypeScript configuration There is a single `tsconfig.base.json` at the repo root with the shared `compilerOptions`. Every package extends it and overrides only what's unique to that package (e.g. `outDir`, JSX runtime, framework lib). diff --git a/nx.json b/nx.json index 6b205bc5d8..c0fcc94775 100644 --- a/nx.json +++ b/nx.json @@ -5,7 +5,7 @@ "useDaemonProcess": true, "useInferencePlugins": false, "analytics": false, - "parallel": 15, + "parallel": 4, "tui": { "enabled": false }, diff --git a/package.json b/package.json index 77d9e58bd0..885581b9b2 100644 --- a/package.json +++ b/package.json @@ -14,13 +14,13 @@ "clean": "pnpm --filter \"./packages/**\" run clean", "clean:all": "git clean -fdx --exclude=\"!.env\"", "test": "pnpm run test:ci", - "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", - "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:pr": "pnpm run test:react-native && VITEST_MAX_WORKERS=1 nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:ci": "pnpm run test:react-native && VITEST_MAX_WORKERS=1 nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", "test:oxlint": "nx affected --target=test:oxlint --exclude=examples/**,testing/**", "test:sherif": "sherif --ignore-dependency typescript", - "test:lib": "nx affected --targets=test:lib --exclude=examples/**,testing/**", + "test:lib": "VITEST_MAX_WORKERS=1 nx affected --targets=test:lib --exclude=examples/**,testing/**", "test:lib:dev": "pnpm test:lib && nx watch --all -- pnpm test:lib", - "test:coverage": "nx affected --targets=test:coverage --exclude=examples/**,testing/**", + "test:coverage": "VITEST_MAX_WORKERS=1 nx affected --targets=test:coverage --exclude=examples/**,testing/**", "test:build": "nx affected --target=test:build --exclude=examples/**,testing/**", "test:types": "nx affected --targets=test:types --exclude=examples/**,testing/**", "test:knip": "knip", diff --git a/packages/ai-react-ui/package.json b/packages/ai-react-ui/package.json index d07b8ed252..1fff95f60a 100644 --- a/packages/ai-react-ui/package.json +++ b/packages/ai-react-ui/package.json @@ -36,8 +36,6 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/ai-solid-ui/package.json b/packages/ai-solid-ui/package.json index 54ff77da96..8c97d1c51d 100644 --- a/packages/ai-solid-ui/package.json +++ b/packages/ai-solid-ui/package.json @@ -37,8 +37,6 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/ai-vue-ui/package.json b/packages/ai-vue-ui/package.json index bf038ab7b2..5c19297953 100644 --- a/packages/ai-vue-ui/package.json +++ b/packages/ai-vue-ui/package.json @@ -34,8 +34,6 @@ "build": "node -e \"process.exit(0)\"", "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/preact-ai-devtools/package.json b/packages/preact-ai-devtools/package.json index 5d0bc3ee25..4884910e96 100644 --- a/packages/preact-ai-devtools/package.json +++ b/packages/preact-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -55,7 +53,6 @@ "@tanstack/devtools-utils": "^0.7.0" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "vite": "^8.2.1" }, "peerDependencies": { diff --git a/packages/preact-ai-devtools/vitest.config.ts b/packages/preact-ai-devtools/vitest.config.ts deleted file mode 100644 index 2f35653f15..0000000000 --- a/packages/preact-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - watch: false, - }, -}) diff --git a/packages/react-ai-devtools/package.json b/packages/react-ai-devtools/package.json index a2951c2ad0..eb8cd89ac0 100644 --- a/packages/react-ai-devtools/package.json +++ b/packages/react-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -60,7 +58,6 @@ }, "devDependencies": { "@types/react": "^19.2.7", - "@vitest/coverage-v8": "4.1.10", "react": "^19.2.3", "vite": "^8.2.1" }, diff --git a/packages/react-ai-devtools/vitest.config.ts b/packages/react-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4fd..0000000000 --- a/packages/react-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/packages/solid-ai-devtools/package.json b/packages/solid-ai-devtools/package.json index 70ce24ca58..14f9b9521c 100644 --- a/packages/solid-ai-devtools/package.json +++ b/packages/solid-ai-devtools/package.json @@ -42,8 +42,6 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ @@ -67,7 +65,6 @@ "solid-js": ">=1.9.7" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "solid-js": "^1.9.10", "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14" diff --git a/packages/solid-ai-devtools/vitest.config.ts b/packages/solid-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4fd..0000000000 --- a/packages/solid-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/packages/svelte-ai-devtools/package.json b/packages/svelte-ai-devtools/package.json index 9ca8180b5e..a5f243487b 100644 --- a/packages/svelte-ai-devtools/package.json +++ b/packages/svelte-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -57,7 +55,6 @@ "svelte": "^5.0.0" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "svelte": "^5.56.9", "vite": "^8.2.1" }, diff --git a/packages/svelte-ai-devtools/vitest.config.ts b/packages/svelte-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4fd..0000000000 --- a/packages/svelte-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da0e7c0f37..eb5164a905 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3025,9 +3025,6 @@ importers: specifier: ^10.0.0 version: 10.28.1 devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) vite: specifier: ^8.2.1 version: 8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) @@ -3044,9 +3041,6 @@ importers: '@types/react': specifier: ^19.2.7 version: 19.2.7 - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) react: specifier: ^19.2.3 version: 19.2.3 @@ -3063,9 +3057,6 @@ importers: specifier: ^0.7.0 version: 0.7.0(@angular/core@21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1))(@types/react@19.2.18)(preact@10.28.2)(react@19.2.3)(solid-js@1.9.10)(svelte@5.56.9(@typescript-eslint/types@8.59.4))(vue@3.5.25(typescript@7.0.2)) devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) solid-js: specifier: ^1.9.10 version: 1.9.10 @@ -3085,9 +3076,6 @@ importers: specifier: ^0.7.0 version: 0.7.0(@angular/core@21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1))(@types/react@19.2.18)(preact@10.28.2)(react@19.2.3)(solid-js@1.9.10)(svelte@5.56.9(@typescript-eslint/types@8.59.4))(vue@3.5.25(typescript@7.0.2)) devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) svelte: specifier: ^5.56.9 version: 5.56.9(@typescript-eslint/types@8.59.4) diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 39338c7c96..6f94881926 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -4,7 +4,9 @@ End-to-end tests for TanStack AI using Playwright and [aimock](https://github.co **Architecture:** Playwright drives a TanStack Start app (`testing/e2e/`) which routes requests through provider adapters pointing at aimock. Fixtures define mock responses. No real API keys needed. All scenarios (including tool execution flows) use aimock fixtures. Tests run in parallel with per-test `X-Test-Id` isolation. -**Providers tested:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare +**Providers tested in CI:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare + +**Local `pnpm test:e2e`** runs openai, anthropic, and gemini. Features that none of those support still run on the providers that do. Use `E2E_PROVIDERS=*` for the full matrix, or `E2E_PROVIDERS=grok` (comma-separated) to pick providers. Playwright does not retry or record video locally; CI retries twice and keeps the video of a failed test. > **Claude Code (`@tanstack/ai-claude-code`) is excluded from the standard matrix.** It's a harness adapter that spawns the Claude Code runtime as a subprocess, so aimock's per-test `X-Test-Id` header isolation can't be injected into its requests. It's covered by unit tests in the package plus a gated live smoke test in `tests/claude-code.spec.ts` — run it with `CLAUDE_CODE_E2E=1` and an `ANTHROPIC_API_KEY` (or a local `claude login`). diff --git a/testing/e2e/playwright.config.ts b/testing/e2e/playwright.config.ts index 51cd66ca7f..b4d7d90fb7 100644 --- a/testing/e2e/playwright.config.ts +++ b/testing/e2e/playwright.config.ts @@ -4,16 +4,18 @@ export default defineConfig({ testDir: './tests', fullyParallel: true, forbidOnly: !!process.env.CI, - retries: 2, - workers: process.env.CI ? 4 : undefined, - reporter: [['html', { open: 'never' }], ['list']], + retries: process.env.CI ? 2 : 0, + workers: 4, + reporter: process.env.CI + ? [['html', { open: 'never' }], ['list']] + : [['list']], timeout: 30_000, expect: { timeout: 15_000, }, use: { baseURL: 'http://localhost:3010', - video: 'on', + video: 'retain-on-failure', screenshot: 'only-on-failure', trace: 'on-first-retry', }, diff --git a/testing/e2e/tests/test-matrix.ts b/testing/e2e/tests/test-matrix.ts index ddfb962dec..5a3848464a 100644 --- a/testing/e2e/tests/test-matrix.ts +++ b/testing/e2e/tests/test-matrix.ts @@ -42,7 +42,45 @@ export const providers: Provider[] = [ export { isSupported } +/** + * Local `pnpm test:e2e` runs these adapter families (OpenAI, Anthropic, + * Gemini). Features that none of them support (TTS, image-gen, …) still + * run against whatever providers do. CI and `E2E_PROVIDERS=*` keep the + * full matrix. Comma-separated ids narrow it further (`E2E_PROVIDERS=grok`). + */ +const LOCAL_E2E_PROVIDERS: ReadonlySet = new Set([ + 'openai', + 'anthropic', + 'gemini', +]) + +function requestedProviders(): 'all' | ReadonlySet { + if (process.env.CI) return 'all' + const raw = process.env.E2E_PROVIDERS + if (raw === undefined || raw.trim() === '') return LOCAL_E2E_PROVIDERS + if (raw.trim() === '*') return 'all' + + const tokens = raw + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + const known = new Set(providers) + const unknown = tokens.filter((t) => !known.has(t)) + if (unknown.length > 0) { + throw new Error( + `E2E_PROVIDERS has unknown provider(s): ${unknown.join(', ')}. Known: ${providers.join(', ')}`, + ) + } + return new Set(providers.filter((p) => tokens.includes(p))) +} + /** Get only the providers that support a given feature */ export function providersFor(feature: Feature): Provider[] { - return providers.filter((p) => isSupported(p, feature)) + const supported = providers.filter((p) => isSupported(p, feature)) + const requested = requestedProviders() + if (requested === 'all') return supported + const filtered = supported.filter((p) => requested.has(p)) + // Feature only exists on providers outside the local default (e.g. TTS + // on elevenlabs). Keep those tests; don't silently drop the feature. + return filtered.length > 0 ? filtered : supported } diff --git a/vitest.workspace.js b/vitest.workspace.js deleted file mode 100644 index 30e867dcec..0000000000 --- a/vitest.workspace.js +++ /dev/null @@ -1,16 +0,0 @@ -// @ts-check - -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - projects: [ - './packages/ai/vite.config.ts', - './packages/ai-lite/vite.config.ts', - './packages/react-ai/vite.config.ts', - './packages/react-ai-devtools/vite.config.ts', - './packages/solid-ai/vite.config.ts', - './packages/solid-ai-devtools/vite.config.ts', - ], - }, -})