diff --git a/.prettierignore b/.prettierignore index 43399fcc..26ebc797 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,3 +3,7 @@ # repo's `prettier --check .` would reformat the file and fight the generator's # output (and the refresh workflow's `git diff` check). src/seps/traceability.json + +# Local tooling workspaces (not part of the repo). +.claude/ +.sdk-under-test/ diff --git a/README.md b/README.md index 333bfcb9..71a8cb2c 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") -- `--suite` - Run a suite of tests in parallel (e.g., "auth") +- `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), or `sep-835` - `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) @@ -81,7 +81,7 @@ npx @modelcontextprotocol/conformance server --url [--scenario ] - `--url` - URL of the server to test - `--scenario ` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default -- `--suite ` - Suite to run: "active" (default), "all", or "pending" +- `--suite ` - Suite to run: "active" (default; excludes pending and draft-spec scenarios), "all", "draft" (scenarios targeting the in-progress draft spec), or "pending" - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--verbose` - Show verbose output diff --git a/src/index.ts b/src/index.ts index a3019f0a..4fd84a51 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,7 @@ import { listExtensionScenarios, listBackcompatScenarios, listDraftScenarios, + listDraftClientScenarios, listScenariosForSpec, listClientScenariosForSpec, getScenarioSpecVersions, @@ -314,7 +315,7 @@ program ) .option( '--suite ', - 'Suite to run: "active" (default, excludes pending), "all", or "pending"', + 'Suite to run: "active" (default, excludes pending and draft), "all", "draft", or "pending"', 'active' ) .option( @@ -378,9 +379,13 @@ program scenarios = listActiveClientScenarios(); } else if (suite === 'pending') { scenarios = listPendingClientScenarios(); + } else if (suite === 'draft') { + // Scenarios targeting the in-progress draft spec; excluded from + // 'active' until the draft is published as a dated release. + scenarios = listDraftClientScenarios(); } else { console.error(`Unknown suite: ${suite}`); - console.error('Available suites: active, all, core, pending'); + console.error('Available suites: active, all, core, draft, pending'); process.exit(1); } diff --git a/src/scenarios/client/auth/spec-references.ts b/src/scenarios/client/auth/spec-references.ts index 908a04eb..681c3e44 100644 --- a/src/scenarios/client/auth/spec-references.ts +++ b/src/scenarios/client/auth/spec-references.ts @@ -104,7 +104,7 @@ export const SpecReferences: { [key: string]: SpecReference } = { }, SEP_990_ENTERPRISE_OAUTH: { id: 'SEP-990-Enterprise-Managed-OAuth', - url: 'https://github.com/modelcontextprotocol/ext-auth/blob/main/specification/draft/enterprise-oauth.mdx' + url: 'https://github.com/modelcontextprotocol/ext-auth/blob/main/specification/draft/enterprise-managed-authorization.mdx' }, SEP_2207_REFRESH_TOKEN_GUIDANCE: { id: 'SEP-2207-Refresh-Token-Guidance', diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 9b54e819..1422c90e 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -206,13 +206,24 @@ const allClientScenariosList: ClientScenario[] = [ new InputRequiredResultValidateInputScenario() ]; -// Active client scenarios (excludes pending) +// Scenarios that test requirements introduced in the in-progress draft spec. +// They run via `--suite draft` (or `--suite all`) and are excluded from the +// default `active` suite until the draft is published as a dated release. +const draftClientScenariosList: ClientScenario[] = + allClientScenariosList.filter( + (scenario) => + 'introducedIn' in scenario.source && + scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION + ); + +// Active client scenarios (excludes pending and draft) const activeClientScenariosList: ClientScenario[] = allClientScenariosList.filter( (scenario) => !pendingClientScenariosList.some( (pending) => pending.name === scenario.name - ) + ) && + !draftClientScenariosList.some((draft) => draft.name === scenario.name) ); // Client scenarios map - built from list @@ -329,8 +340,21 @@ export function listClientScenariosForAuthorizationServer(): string[] { return Array.from(clientScenariosForAuthorizationServer.keys()); } +// All client-testing scenarios that target the draft spec, derived from the +// declared `source.introducedIn` rather than a hand-maintained list (covers +// both the auth draft scenarios and the non-auth ones, e.g. SEP-2243/2575). +const draftSpecScenariosList: Scenario[] = scenariosList.filter( + (scenario) => + 'introducedIn' in scenario.source && + scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION +); + export function listDraftScenarios(): string[] { - return draftScenariosList.map((scenario) => scenario.name); + return draftSpecScenariosList.map((scenario) => scenario.name); +} + +export function listDraftClientScenarios(): string[] { + return draftClientScenariosList.map((scenario) => scenario.name); } export { listMetadataScenarios }; diff --git a/src/scenarios/server/all-scenarios.test.ts b/src/scenarios/server/all-scenarios.test.ts index 9f2d014e..bc697a3b 100644 --- a/src/scenarios/server/all-scenarios.test.ts +++ b/src/scenarios/server/all-scenarios.test.ts @@ -1,6 +1,11 @@ import { spawn, ChildProcess } from 'child_process'; import { createServer } from 'net'; -import { getClientScenario, listActiveClientScenarios } from '../index'; +import { + getClientScenario, + listActiveClientScenarios, + listDraftClientScenarios, + listPendingClientScenarios +} from '../index'; import path from 'path'; function getFreePort(): Promise { @@ -116,8 +121,14 @@ describe('Server Scenarios', () => { } }); - // Generate individual test for each scenario - const scenarios = listActiveClientScenarios(); + // Generate individual test for each scenario: the active suite plus the + // draft-spec scenarios that aren't parked in `pending` — the same set this + // file covered before draft scenarios were split out of `active`. + const pendingScenarios = new Set(listPendingClientScenarios()); + const scenarios = [ + ...listActiveClientScenarios(), + ...listDraftClientScenarios().filter((name) => !pendingScenarios.has(name)) + ]; for (const scenarioName of scenarios) { it(`${scenarioName}`, async () => { diff --git a/src/scenarios/server/tools.ts b/src/scenarios/server/tools.ts index 2fb4717a..5875e725 100644 --- a/src/scenarios/server/tools.ts +++ b/src/scenarios/server/tools.ts @@ -21,7 +21,7 @@ const TOOLS_NAME_FORMAT_SPEC_REFS = [ }, { id: 'SEP-986', - url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/SEP/SEP-986.md' + url: 'https://modelcontextprotocol.io/specification/2025-11-25/server/tools#tool-names' } ]; diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index c77e43e7..93b46620 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -3,11 +3,14 @@ import { listScenarios, listScenariosForSpec, listDraftScenarios, + listDraftClientScenarios, + listActiveClientScenarios, listExtensionScenarios, getScenarioSpecVersions, resolveSpecVersion, ALL_SPEC_VERSIONS, - scenarios + scenarios, + clientScenarios } from './index'; import { DATED_SPEC_VERSIONS, @@ -94,3 +97,51 @@ describe('specVersions helpers', () => { } }); }); + +describe('draft suite membership', () => { + it('every scenario introduced in the draft spec is selected by its draft suite', () => { + const draftClientTesting = new Set(listDraftScenarios()); + for (const [name, scenario] of scenarios) { + if ( + 'introducedIn' in scenario.source && + scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION + ) { + expect( + draftClientTesting.has(name), + `client-testing scenario "${name}" should be in the draft suite` + ).toBe(true); + } + } + + const draftServerTesting = new Set(listDraftClientScenarios()); + for (const [name, scenario] of clientScenarios) { + if ( + 'introducedIn' in scenario.source && + scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION + ) { + expect( + draftServerTesting.has(name), + `server-testing scenario "${name}" should be in the draft suite` + ).toBe(true); + } + } + }); + + it('the draft suite covers the non-auth draft client scenarios', () => { + const draft = new Set(listDraftScenarios()); + expect(draft.has('request-metadata')).toBe(true); + expect(draft.has('http-standard-headers')).toBe(true); + expect(draft.has('sep-2322-client-request-state')).toBe(true); + }); + + it('draft server-testing scenarios are excluded from the active suite', () => { + const active = new Set(listActiveClientScenarios()); + expect(listDraftClientScenarios().length).toBeGreaterThan(0); + for (const name of listDraftClientScenarios()) { + expect( + active.has(name), + `draft scenario "${name}" should not be in the active suite` + ).toBe(false); + } + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index b0f36f97..bd0c67ab 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,7 +5,15 @@ export default defineConfig({ globals: true, environment: 'node', include: ['**/*.test.ts'], - exclude: ['**/node_modules/**', 'dist', '.sdk-under-test'], + exclude: [ + '**/node_modules/**', + 'dist', + '.sdk-under-test', + // Local tooling workspaces (Claude worktrees, SDK checkouts) must never + // be collected as test files. + '**/.claude/**', + '**/.sdk-under-test/**' + ], // Run test files sequentially to avoid port conflicts fileParallelism: false, // Increase timeout for server tests in CI