diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c051e15e0..5aefd95f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,6 +92,26 @@ All commands are run from the repo root. Nx handles affected detection and cachi Working on a single package? `cd packages/` and use its scripts directly (`pnpm test:lib`, `pnpm test:types`, etc.). +### Faster local runs + +Root `pnpm test`, `pnpm test:pr`, and `pnpm test:lib` set `VITEST_MAX_WORKERS=1`. Nx runs 4 tasks at a time (`nx.json` `parallel`). Together that stops each package from spawning one Vitest worker per CPU core while 15 packages already run in parallel. + +A single package still uses all cores: + +```bash +cd packages/ai +pnpm test:lib +``` + +Local `pnpm test:e2e` runs openai, anthropic, and gemini. Features that none of those support (TTS, image-gen, and similar) still run on the providers that do. CI always runs the full matrix. + +```bash +E2E_PROVIDERS=* pnpm test:e2e # full matrix locally +E2E_PROVIDERS=grok pnpm test:e2e # one provider +``` + +Playwright does not retry or record video locally. CI retries twice and keeps the video of a failed test. + ## TypeScript configuration There is a single `tsconfig.base.json` at the repo root with the shared `compilerOptions`. Every package extends it and overrides only what's unique to that package (e.g. `outDir`, JSX runtime, framework lib). diff --git a/nx.json b/nx.json index 6b205bc5d..c0fcc9477 100644 --- a/nx.json +++ b/nx.json @@ -5,7 +5,7 @@ "useDaemonProcess": true, "useInferencePlugins": false, "analytics": false, - "parallel": 15, + "parallel": 4, "tui": { "enabled": false }, diff --git a/package.json b/package.json index 77d9e58bd..885581b9b 100644 --- a/package.json +++ b/package.json @@ -14,13 +14,13 @@ "clean": "pnpm --filter \"./packages/**\" run clean", "clean:all": "git clean -fdx --exclude=\"!.env\"", "test": "pnpm run test:ci", - "test:pr": "pnpm run test:react-native && nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", - "test:ci": "pnpm run test:react-native && nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:pr": "pnpm run test:react-native && VITEST_MAX_WORKERS=1 nx affected --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", + "test:ci": "pnpm run test:react-native && VITEST_MAX_WORKERS=1 nx run-many --targets=test:sherif,test:knip,test:docs,test:kiira,test:maintainer,test:ai-review,test:oxlint,test:lib,test:types,test:build,build && pnpm test:dts", "test:oxlint": "nx affected --target=test:oxlint --exclude=examples/**,testing/**", "test:sherif": "sherif --ignore-dependency typescript", - "test:lib": "nx affected --targets=test:lib --exclude=examples/**,testing/**", + "test:lib": "VITEST_MAX_WORKERS=1 nx affected --targets=test:lib --exclude=examples/**,testing/**", "test:lib:dev": "pnpm test:lib && nx watch --all -- pnpm test:lib", - "test:coverage": "nx affected --targets=test:coverage --exclude=examples/**,testing/**", + "test:coverage": "VITEST_MAX_WORKERS=1 nx affected --targets=test:coverage --exclude=examples/**,testing/**", "test:build": "nx affected --target=test:build --exclude=examples/**,testing/**", "test:types": "nx affected --targets=test:types --exclude=examples/**,testing/**", "test:knip": "knip", diff --git a/packages/ai-react-ui/package.json b/packages/ai-react-ui/package.json index d07b8ed25..1fff95f60 100644 --- a/packages/ai-react-ui/package.json +++ b/packages/ai-react-ui/package.json @@ -36,8 +36,6 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/ai-solid-ui/package.json b/packages/ai-solid-ui/package.json index 54ff77da9..8c97d1c51 100644 --- a/packages/ai-solid-ui/package.json +++ b/packages/ai-solid-ui/package.json @@ -37,8 +37,6 @@ "clean": "premove ./build ./dist", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/ai-vue-ui/package.json b/packages/ai-vue-ui/package.json index bf038ab7b..5c1929795 100644 --- a/packages/ai-vue-ui/package.json +++ b/packages/ai-vue-ui/package.json @@ -34,8 +34,6 @@ "build": "node -e \"process.exit(0)\"", "clean": "premove ./build ./dist", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ diff --git a/packages/preact-ai-devtools/package.json b/packages/preact-ai-devtools/package.json index 5d0bc3ee2..4884910e9 100644 --- a/packages/preact-ai-devtools/package.json +++ b/packages/preact-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -55,7 +53,6 @@ "@tanstack/devtools-utils": "^0.7.0" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "vite": "^8.2.1" }, "peerDependencies": { diff --git a/packages/preact-ai-devtools/vitest.config.ts b/packages/preact-ai-devtools/vitest.config.ts deleted file mode 100644 index 2f35653f1..000000000 --- a/packages/preact-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - watch: false, - }, -}) diff --git a/packages/react-ai-devtools/package.json b/packages/react-ai-devtools/package.json index a2951c2ad..eb8cd89ac 100644 --- a/packages/react-ai-devtools/package.json +++ b/packages/react-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -60,7 +58,6 @@ }, "devDependencies": { "@types/react": "^19.2.7", - "@vitest/coverage-v8": "4.1.10", "react": "^19.2.3", "vite": "^8.2.1" }, diff --git a/packages/react-ai-devtools/vitest.config.ts b/packages/react-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4f..000000000 --- a/packages/react-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/packages/solid-ai-devtools/package.json b/packages/solid-ai-devtools/package.json index 70ce24ca5..14f9b9521 100644 --- a/packages/solid-ai-devtools/package.json +++ b/packages/solid-ai-devtools/package.json @@ -42,8 +42,6 @@ "lint:fix": "oxlint src --type-aware --fix", "test:build": "publint --strict", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc" }, "keywords": [ @@ -67,7 +65,6 @@ "solid-js": ">=1.9.7" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "solid-js": "^1.9.10", "vite": "^8.2.1", "vite-plugin-solid": "^2.11.14" diff --git a/packages/solid-ai-devtools/vitest.config.ts b/packages/solid-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4f..000000000 --- a/packages/solid-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/packages/svelte-ai-devtools/package.json b/packages/svelte-ai-devtools/package.json index 9ca8180b5..a5f243487 100644 --- a/packages/svelte-ai-devtools/package.json +++ b/packages/svelte-ai-devtools/package.json @@ -21,8 +21,6 @@ "clean": "premove ./build ./dist", "lint:fix": "oxlint src --type-aware --fix", "test:oxlint": "oxlint src --type-aware", - "test:lib": "vitest --passWithNoTests", - "test:lib:dev": "pnpm test:lib --watch", "test:types": "tsc", "test:build": "publint --strict", "build": "vite build" @@ -57,7 +55,6 @@ "svelte": "^5.0.0" }, "devDependencies": { - "@vitest/coverage-v8": "4.1.10", "svelte": "^5.56.9", "vite": "^8.2.1" }, diff --git a/packages/svelte-ai-devtools/vitest.config.ts b/packages/svelte-ai-devtools/vitest.config.ts deleted file mode 100644 index 57223ef4f..000000000 --- a/packages/svelte-ai-devtools/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - globals: true, - environment: 'node', - include: ['tests/**/*.test.ts'], - }, -}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da0e7c0f3..eb5164a90 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3025,9 +3025,6 @@ importers: specifier: ^10.0.0 version: 10.28.1 devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) vite: specifier: ^8.2.1 version: 8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) @@ -3044,9 +3041,6 @@ importers: '@types/react': specifier: ^19.2.7 version: 19.2.7 - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) react: specifier: ^19.2.3 version: 19.2.3 @@ -3063,9 +3057,6 @@ importers: specifier: ^0.7.0 version: 0.7.0(@angular/core@21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1))(@types/react@19.2.18)(preact@10.28.2)(react@19.2.3)(solid-js@1.9.10)(svelte@5.56.9(@typescript-eslint/types@8.59.4))(vue@3.5.25(typescript@7.0.2)) devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) solid-js: specifier: ^1.9.10 version: 1.9.10 @@ -3085,9 +3076,6 @@ importers: specifier: ^0.7.0 version: 0.7.0(@angular/core@21.2.20(@angular/compiler@21.2.20)(rxjs@7.8.2)(zone.js@0.15.1))(@types/react@19.2.18)(preact@10.28.2)(react@19.2.3)(solid-js@1.9.10)(svelte@5.56.9(@typescript-eslint/types@8.59.4))(vue@3.5.25(typescript@7.0.2)) devDependencies: - '@vitest/coverage-v8': - specifier: 4.1.10 - version: 4.1.10(vitest@4.1.10) svelte: specifier: ^5.56.9 version: 5.56.9(@typescript-eslint/types@8.59.4) diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 39338c7c9..6f9488192 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -4,7 +4,9 @@ End-to-end tests for TanStack AI using Playwright and [aimock](https://github.co **Architecture:** Playwright drives a TanStack Start app (`testing/e2e/`) which routes requests through provider adapters pointing at aimock. Fixtures define mock responses. No real API keys needed. All scenarios (including tool execution flows) use aimock fixtures. Tests run in parallel with per-test `X-Test-Id` isolation. -**Providers tested:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare +**Providers tested in CI:** openai, anthropic, gemini, vertex, vertex-grok, vertex-mistral, ollama, groq, grok, openrouter, openrouter-responses, vercel-gateway, vercel-gateway-responses, lovable, lovable-responses, bedrock, bedrock-responses, openai-compatible, openai-compatible-legacy, mistral, byteplus, elevenlabs, llmgateway, cloudflare + +**Local `pnpm test:e2e`** runs openai, anthropic, and gemini. Features that none of those support still run on the providers that do. Use `E2E_PROVIDERS=*` for the full matrix, or `E2E_PROVIDERS=grok` (comma-separated) to pick providers. Playwright does not retry or record video locally; CI retries twice and keeps the video of a failed test. > **Claude Code (`@tanstack/ai-claude-code`) is excluded from the standard matrix.** It's a harness adapter that spawns the Claude Code runtime as a subprocess, so aimock's per-test `X-Test-Id` header isolation can't be injected into its requests. It's covered by unit tests in the package plus a gated live smoke test in `tests/claude-code.spec.ts` — run it with `CLAUDE_CODE_E2E=1` and an `ANTHROPIC_API_KEY` (or a local `claude login`). diff --git a/testing/e2e/playwright.config.ts b/testing/e2e/playwright.config.ts index 51cd66ca7..b4d7d90fb 100644 --- a/testing/e2e/playwright.config.ts +++ b/testing/e2e/playwright.config.ts @@ -4,16 +4,18 @@ export default defineConfig({ testDir: './tests', fullyParallel: true, forbidOnly: !!process.env.CI, - retries: 2, - workers: process.env.CI ? 4 : undefined, - reporter: [['html', { open: 'never' }], ['list']], + retries: process.env.CI ? 2 : 0, + workers: 4, + reporter: process.env.CI + ? [['html', { open: 'never' }], ['list']] + : [['list']], timeout: 30_000, expect: { timeout: 15_000, }, use: { baseURL: 'http://localhost:3010', - video: 'on', + video: 'retain-on-failure', screenshot: 'only-on-failure', trace: 'on-first-retry', }, diff --git a/testing/e2e/tests/test-matrix.ts b/testing/e2e/tests/test-matrix.ts index ddfb962de..5a3848464 100644 --- a/testing/e2e/tests/test-matrix.ts +++ b/testing/e2e/tests/test-matrix.ts @@ -42,7 +42,45 @@ export const providers: Provider[] = [ export { isSupported } +/** + * Local `pnpm test:e2e` runs these adapter families (OpenAI, Anthropic, + * Gemini). Features that none of them support (TTS, image-gen, …) still + * run against whatever providers do. CI and `E2E_PROVIDERS=*` keep the + * full matrix. Comma-separated ids narrow it further (`E2E_PROVIDERS=grok`). + */ +const LOCAL_E2E_PROVIDERS: ReadonlySet = new Set([ + 'openai', + 'anthropic', + 'gemini', +]) + +function requestedProviders(): 'all' | ReadonlySet { + if (process.env.CI) return 'all' + const raw = process.env.E2E_PROVIDERS + if (raw === undefined || raw.trim() === '') return LOCAL_E2E_PROVIDERS + if (raw.trim() === '*') return 'all' + + const tokens = raw + .split(',') + .map((s) => s.trim()) + .filter(Boolean) + const known = new Set(providers) + const unknown = tokens.filter((t) => !known.has(t)) + if (unknown.length > 0) { + throw new Error( + `E2E_PROVIDERS has unknown provider(s): ${unknown.join(', ')}. Known: ${providers.join(', ')}`, + ) + } + return new Set(providers.filter((p) => tokens.includes(p))) +} + /** Get only the providers that support a given feature */ export function providersFor(feature: Feature): Provider[] { - return providers.filter((p) => isSupported(p, feature)) + const supported = providers.filter((p) => isSupported(p, feature)) + const requested = requestedProviders() + if (requested === 'all') return supported + const filtered = supported.filter((p) => requested.has(p)) + // Feature only exists on providers outside the local default (e.g. TTS + // on elevenlabs). Keep those tests; don't silently drop the feature. + return filtered.length > 0 ? filtered : supported } diff --git a/vitest.workspace.js b/vitest.workspace.js deleted file mode 100644 index 30e867dce..000000000 --- a/vitest.workspace.js +++ /dev/null @@ -1,16 +0,0 @@ -// @ts-check - -import { defineConfig } from 'vitest/config' - -export default defineConfig({ - test: { - projects: [ - './packages/ai/vite.config.ts', - './packages/ai-lite/vite.config.ts', - './packages/react-ai/vite.config.ts', - './packages/react-ai-devtools/vite.config.ts', - './packages/solid-ai/vite.config.ts', - './packages/solid-ai-devtools/vite.config.ts', - ], - }, -})