Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,7 @@
# repo's `prettier --check .` would reformat the file and fight the generator's
# output (and the refresh workflow's `git diff` check).
src/seps/traceability.json

# Local tooling workspaces (not part of the repo).
.claude/
.sdk-under-test/
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen

- `--command` - The command to run your MCP client (can include flags)
- `--scenario` - The test scenario to run (e.g., "initialize")
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
- `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), or `sep-835`
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
- `--timeout` - Timeout in milliseconds (default: 30000)
Expand All @@ -81,7 +81,7 @@ npx @modelcontextprotocol/conformance server --url <url> [--scenario <scenario>]

- `--url` - URL of the server to test
- `--scenario <scenario>` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default
- `--suite <suite>` - Suite to run: "active" (default), "all", or "pending"
- `--suite <suite>` - Suite to run: "active" (default; excludes pending and draft-spec scenarios), "all", "draft" (scenarios targeting the in-progress draft spec), or "pending"
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
- `--verbose` - Show verbose output

Expand Down
9 changes: 7 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
listExtensionScenarios,
listBackcompatScenarios,
listDraftScenarios,
listDraftClientScenarios,
listScenariosForSpec,
listClientScenariosForSpec,
getScenarioSpecVersions,
Expand Down Expand Up @@ -314,7 +315,7 @@ program
)
.option(
'--suite <suite>',
'Suite to run: "active" (default, excludes pending), "all", or "pending"',
'Suite to run: "active" (default, excludes pending and draft), "all", "draft", or "pending"',
'active'
)
.option(
Expand Down Expand Up @@ -378,9 +379,13 @@ program
scenarios = listActiveClientScenarios();
} else if (suite === 'pending') {
scenarios = listPendingClientScenarios();
} else if (suite === 'draft') {
// Scenarios targeting the in-progress draft spec; excluded from
// 'active' until the draft is published as a dated release.
scenarios = listDraftClientScenarios();
} else {
console.error(`Unknown suite: ${suite}`);
console.error('Available suites: active, all, core, pending');
console.error('Available suites: active, all, core, draft, pending');
process.exit(1);
}

Expand Down
2 changes: 1 addition & 1 deletion src/scenarios/client/auth/spec-references.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ export const SpecReferences: { [key: string]: SpecReference } = {
},
SEP_990_ENTERPRISE_OAUTH: {
id: 'SEP-990-Enterprise-Managed-OAuth',
url: 'https://github.com/modelcontextprotocol/ext-auth/blob/main/specification/draft/enterprise-oauth.mdx'
url: 'https://github.com/modelcontextprotocol/ext-auth/blob/main/specification/draft/enterprise-managed-authorization.mdx'
},
SEP_2207_REFRESH_TOKEN_GUIDANCE: {
id: 'SEP-2207-Refresh-Token-Guidance',
Expand Down
30 changes: 27 additions & 3 deletions src/scenarios/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,13 +206,24 @@ const allClientScenariosList: ClientScenario[] = [
new InputRequiredResultValidateInputScenario()
];

// Active client scenarios (excludes pending)
// Scenarios that test requirements introduced in the in-progress draft spec.
// They run via `--suite draft` (or `--suite all`) and are excluded from the
// default `active` suite until the draft is published as a dated release.
const draftClientScenariosList: ClientScenario[] =
allClientScenariosList.filter(
(scenario) =>
'introducedIn' in scenario.source &&
scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION
);

// Active client scenarios (excludes pending and draft)
const activeClientScenariosList: ClientScenario[] =
allClientScenariosList.filter(
(scenario) =>
!pendingClientScenariosList.some(
(pending) => pending.name === scenario.name
)
) &&
!draftClientScenariosList.some((draft) => draft.name === scenario.name)
);

// Client scenarios map - built from list
Expand Down Expand Up @@ -329,8 +340,21 @@ export function listClientScenariosForAuthorizationServer(): string[] {
return Array.from(clientScenariosForAuthorizationServer.keys());
}

// All client-testing scenarios that target the draft spec, derived from the
// declared `source.introducedIn` rather than a hand-maintained list (covers
// both the auth draft scenarios and the non-auth ones, e.g. SEP-2243/2575).
const draftSpecScenariosList: Scenario[] = scenariosList.filter(
(scenario) =>
'introducedIn' in scenario.source &&
scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION
);

export function listDraftScenarios(): string[] {
return draftScenariosList.map((scenario) => scenario.name);
return draftSpecScenariosList.map((scenario) => scenario.name);
}

export function listDraftClientScenarios(): string[] {
return draftClientScenariosList.map((scenario) => scenario.name);
}

export { listMetadataScenarios };
Expand Down
17 changes: 14 additions & 3 deletions src/scenarios/server/all-scenarios.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import { spawn, ChildProcess } from 'child_process';
import { createServer } from 'net';
import { getClientScenario, listActiveClientScenarios } from '../index';
import {
getClientScenario,
listActiveClientScenarios,
listDraftClientScenarios,
listPendingClientScenarios
} from '../index';
import path from 'path';

function getFreePort(): Promise<number> {
Expand Down Expand Up @@ -116,8 +121,14 @@ describe('Server Scenarios', () => {
}
});

// Generate individual test for each scenario
const scenarios = listActiveClientScenarios();
// Generate individual test for each scenario: the active suite plus the
// draft-spec scenarios that aren't parked in `pending` — the same set this
// file covered before draft scenarios were split out of `active`.
const pendingScenarios = new Set(listPendingClientScenarios());
const scenarios = [
...listActiveClientScenarios(),
...listDraftClientScenarios().filter((name) => !pendingScenarios.has(name))
];

for (const scenarioName of scenarios) {
it(`${scenarioName}`, async () => {
Expand Down
2 changes: 1 addition & 1 deletion src/scenarios/server/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const TOOLS_NAME_FORMAT_SPEC_REFS = [
},
{
id: 'SEP-986',
url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/SEP/SEP-986.md'
url: 'https://modelcontextprotocol.io/specification/2025-11-25/server/tools#tool-names'
}
];

Expand Down
53 changes: 52 additions & 1 deletion src/scenarios/spec-version.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ import {
listScenarios,
listScenariosForSpec,
listDraftScenarios,
listDraftClientScenarios,
listActiveClientScenarios,
listExtensionScenarios,
getScenarioSpecVersions,
resolveSpecVersion,
ALL_SPEC_VERSIONS,
scenarios
scenarios,
clientScenarios
} from './index';
import {
DATED_SPEC_VERSIONS,
Expand Down Expand Up @@ -94,3 +97,51 @@ describe('specVersions helpers', () => {
}
});
});

describe('draft suite membership', () => {
it('every scenario introduced in the draft spec is selected by its draft suite', () => {
const draftClientTesting = new Set(listDraftScenarios());
for (const [name, scenario] of scenarios) {
if (
'introducedIn' in scenario.source &&
scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION
) {
expect(
draftClientTesting.has(name),
`client-testing scenario "${name}" should be in the draft suite`
).toBe(true);
}
}

const draftServerTesting = new Set(listDraftClientScenarios());
for (const [name, scenario] of clientScenarios) {
if (
'introducedIn' in scenario.source &&
scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION
) {
expect(
draftServerTesting.has(name),
`server-testing scenario "${name}" should be in the draft suite`
).toBe(true);
}
}
});

it('the draft suite covers the non-auth draft client scenarios', () => {
const draft = new Set(listDraftScenarios());
expect(draft.has('request-metadata')).toBe(true);
expect(draft.has('http-standard-headers')).toBe(true);
expect(draft.has('sep-2322-client-request-state')).toBe(true);
});

it('draft server-testing scenarios are excluded from the active suite', () => {
const active = new Set(listActiveClientScenarios());
expect(listDraftClientScenarios().length).toBeGreaterThan(0);
for (const name of listDraftClientScenarios()) {
expect(
active.has(name),
`draft scenario "${name}" should not be in the active suite`
).toBe(false);
}
});
});
10 changes: 9 additions & 1 deletion vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,15 @@ export default defineConfig({
globals: true,
environment: 'node',
include: ['**/*.test.ts'],
exclude: ['**/node_modules/**', 'dist', '.sdk-under-test'],
exclude: [
'**/node_modules/**',
'dist',
'.sdk-under-test',
// Local tooling workspaces (Claude worktrees, SDK checkouts) must never
// be collected as test files.
'**/.claude/**',
'**/.sdk-under-test/**'
],
// Run test files sequentially to avoid port conflicts
fileParallelism: false,
// Increase timeout for server tests in CI
Expand Down
Loading