From d2f46bc9ab3c3c2e39635b416228e3811f260cc9 Mon Sep 17 00:00:00 2001 From: Den Delimarsky Date: Mon, 18 May 2026 07:31:32 +0000 Subject: [PATCH 01/13] Add SEP-2350 scope-accumulation check to auth/scope-step-up SEP-2350 clarifies that on step-up re-authorization, clients SHOULD compute the union of previously-granted and newly-challenged scopes so they don't lose permissions for other operations. - Traceability yaml: 1 client check (two spec sentences merged), 1 server check tracked for a future server-side scenario, 1 excluded reword - ScopeStepUpAuthScenario now challenges with only the missing scope so union accumulation is observable on the second authorization request - New check sep-2350-scope-union-on-reauth (WARNING when the previously granted scope is dropped) - everything-client withOAuthRetry now accumulates prior token scope into the re-auth scope (passing example) - New auth-test-echo-scope negative client + vitest case --- .../typescript/auth-test-echo-scope.ts | 62 +++++++++++++++++++ .../typescript/helpers/withOAuthRetry.ts | 22 ++++++- src/scenarios/client/auth/index.test.ts | 13 +++- src/scenarios/client/auth/scope-handling.ts | 58 +++++++++++++---- src/seps/sep-2350.yaml | 11 ++++ 5 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 examples/clients/typescript/auth-test-echo-scope.ts create mode 100644 src/seps/sep-2350.yaml diff --git a/examples/clients/typescript/auth-test-echo-scope.ts b/examples/clients/typescript/auth-test-echo-scope.ts new file mode 100644 index 00000000..7a197c64 --- /dev/null +++ b/examples/clients/typescript/auth-test-echo-scope.ts @@ -0,0 +1,62 @@ +/** + * Negative client for SEP-2350: on a step-up 403, re-authorizes with ONLY the + * scope from the challenge (does not accumulate previously-granted scopes). + * Expected to trigger sep-2350-scope-union-on-reauth = WARNING. + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { + auth, + extractWWWAuthenticateParams +} from '@modelcontextprotocol/sdk/client/auth.js'; +import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js'; +import { withOAuthRetry } from './helpers/withOAuthRetry'; +import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider'; +import { runAsCli } from './helpers/cliRunner'; + +// handle401 variant that ECHOES the challenge scope verbatim (no SEP-2350 union). +const handle401EchoScope = async ( + response: Response, + provider: ConformanceOAuthProvider, + next: FetchLike, + serverUrl: string | URL +): Promise => { + const { resourceMetadataUrl, scope } = extractWWWAuthenticateParams(response); + let result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + fetchFn: next + }); + if (result === 'REDIRECT') { + const authorizationCode = await provider.getAuthCode(); + result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + authorizationCode, + fetchFn: next + }); + } +}; + +export async function runClient(serverUrl: string): Promise { + const client = new Client( + { name: 'auth-test-echo-scope', version: '1.0.0' }, + { capabilities: {} } + ); + const oauthFetch = withOAuthRetry( + 'auth-test-echo-scope', + new URL(serverUrl), + handle401EchoScope + )(fetch); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + await client.connect(transport); + await client.listTools(); + await client.callTool({ name: 'test-tool', arguments: {} }); + await transport.close(); +} + +runAsCli(runClient, import.meta.url, 'auth-test-echo-scope '); diff --git a/examples/clients/typescript/helpers/withOAuthRetry.ts b/examples/clients/typescript/helpers/withOAuthRetry.ts index 429ca539..d9e19c3d 100644 --- a/examples/clients/typescript/helpers/withOAuthRetry.ts +++ b/examples/clients/typescript/helpers/withOAuthRetry.ts @@ -7,13 +7,33 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js'; import type { Middleware } from '@modelcontextprotocol/sdk/client/middleware.js'; import { ConformanceOAuthProvider } from './ConformanceOAuthProvider'; +/** + * SEP-2350: a well-behaved client computes the union of previously-granted + * scopes and the newly challenged scopes when re-authorizing, so it doesn't + * lose permissions needed for other operations. + */ +const unionScopes = ( + prior: string | undefined, + challenged: string | undefined +): string | undefined => { + const set = new Set( + [...(prior?.split(' ') ?? []), ...(challenged?.split(' ') ?? [])].filter( + Boolean + ) + ); + return set.size > 0 ? [...set].join(' ') : undefined; +}; + export const handle401 = async ( response: Response, provider: ConformanceOAuthProvider, next: FetchLike, serverUrl: string | URL ): Promise => { - const { resourceMetadataUrl, scope } = extractWWWAuthenticateParams(response); + const { resourceMetadataUrl, scope: challengedScope } = + extractWWWAuthenticateParams(response); + const prior = (await provider.tokens())?.scope; + const scope = unionScopes(prior, challengedScope); let result = await auth(provider, { serverUrl, resourceMetadataUrl, diff --git a/src/scenarios/client/auth/index.test.ts b/src/scenarios/client/auth/index.test.ts index e18a4671..cd1066dc 100644 --- a/src/scenarios/client/auth/index.test.ts +++ b/src/scenarios/client/auth/index.test.ts @@ -14,6 +14,7 @@ import { runClient as partialScopesClient } from '../../../../examples/clients/t import { runClient as ignore403Client } from '../../../../examples/clients/typescript/auth-test-ignore-403'; import { runClient as noRetryLimitClient } from '../../../../examples/clients/typescript/auth-test-no-retry-limit'; import { runClient as noPkceClient } from '../../../../examples/clients/typescript/auth-test-no-pkce'; +import { runClient as echoScopeClient } from '../../../../examples/clients/typescript/auth-test-echo-scope'; import { getHandler } from '../../../../examples/clients/typescript/everything-client'; import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger'; @@ -116,7 +117,17 @@ describe('Negative tests', () => { test('client only responds to 401, not 403', async () => { const runner = new InlineClientRunner(ignore403Client); await runClientAgainstScenario(runner, 'auth/scope-step-up', { - expectedFailureSlugs: ['scope-step-up-escalation'] + expectedFailureSlugs: [ + 'scope-step-up-escalation', + 'sep-2350-scope-union-on-reauth' + ] + }); + }); + + test('client echoes challenge scope without accumulating prior grant (SEP-2350)', async () => { + const runner = new InlineClientRunner(echoScopeClient); + await runClientAgainstScenario(runner, 'auth/scope-step-up', { + expectedFailureSlugs: ['sep-2350-scope-union-on-reauth'] }); }); diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index e1d0ff12..9b6c9f68 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -295,7 +295,11 @@ export class ScopeStepUpAuthScenario implements Scenario { this.checks = []; const initialScope = 'mcp:basic'; - const escalatedScopes = ['mcp:basic', 'mcp:write']; + // tools/call gates on mcp:write only (not the union) so the scenario can + // complete even for clients that don't accumulate; the SEP-2350 check then + // observes whether the previously-granted mcp:basic was retained. + const stepUpScope = 'mcp:write'; + const escalatedScopes = [initialScope, stepUpScope]; const tokenVerifier = new MockTokenVerifier(this.checks, escalatedScopes); let authRequestCount = 0; @@ -323,21 +327,40 @@ export class ScopeStepUpAuthScenario implements Scenario { } }); } else if (authRequestCount === 2) { - // Second auth request - should escalate to mcp:basic + mcp:write - const hasAllScopes = escalatedScopes.every((s) => - requestedScopes.includes(s) - ); + // Second auth request after a 403 challenge that listed only the + // *missing* scope (mcp:write). Two distinct assertions: + // - escalation: client included the challenged scope at all + // - SEP-2350 union: client *also* kept the previously-granted scope + // that was NOT in the challenge (i.e., computed prior ∪ challenge) + const includesChallenged = requestedScopes.includes(stepUpScope); this.checks.push({ id: 'scope-step-up-escalation', name: 'Client scope escalation for step-up auth', - description: hasAllScopes - ? 'Client correctly escalated scopes for step-up authentication' + description: includesChallenged + ? 'Client correctly requested the challenged scope for step-up authentication' : 'Client SHOULD request additional scopes when receiving 403 with new scope requirements', - status: hasAllScopes ? 'SUCCESS' : 'WARNING', + status: includesChallenged ? 'SUCCESS' : 'WARNING', timestamp: data.timestamp, specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY], details: { - expectedScopes: escalatedScopes.join(' '), + challengedScope: stepUpScope, + requestedScope: data.scope || 'none' + } + }); + + const retainedPrior = requestedScopes.includes(initialScope); + this.checks.push({ + id: 'sep-2350-scope-union-on-reauth', + name: 'Client accumulates previously-granted scopes on re-authorization', + description: retainedPrior + ? 'Client included previously-granted scopes alongside the newly challenged scope when re-authorizing' + : 'Client SHOULD compute the union of previously requested scopes and newly challenged scopes when initiating re-authorization (SEP-2350); previously-granted scope was dropped', + status: retainedPrior ? 'SUCCESS' : 'WARNING', + timestamp: data.timestamp, + specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING], + details: { + previouslyGranted: initialScope, + challengedScope: stepUpScope, requestedScope: data.scope || 'none' } }); @@ -388,19 +411,21 @@ export class ScopeStepUpAuthScenario implements Scenario { // Determine required scopes based on method const isToolCall = method === 'tools/call'; - const requiredScopes = isToolCall ? escalatedScopes : [initialScope]; + const requiredScopes = isToolCall ? [stepUpScope] : [initialScope]; const hasRequiredScopes = requiredScopes.every((s) => tokenScopes.includes(s) ); if (!hasRequiredScopes) { - // Has token but insufficient scopes - return 403 + // Has token but insufficient scopes - return 403. Challenge with only + // the step-up scope so SEP-2350 union accumulation is observable: a + // client that just echoes the challenge would drop mcp:basic. return res .status(403) .set( 'WWW-Authenticate', - `Bearer scope="${requiredScopes.join(' ')}", resource_metadata="${resourceMetadataUrl()}", error="insufficient_scope"` + `Bearer scope="${stepUpScope}", resource_metadata="${resourceMetadataUrl()}", error="insufficient_scope"` ) .json({ error: 'insufficient_scope', @@ -465,6 +490,15 @@ export class ScopeStepUpAuthScenario implements Scenario { timestamp: new Date().toISOString(), specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] }); + this.checks.push({ + id: 'sep-2350-scope-union-on-reauth', + name: 'Client accumulates previously-granted scopes on re-authorization', + description: + 'Client did not make a second authorization request - scope union check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING] + }); } return this.checks; diff --git a/src/seps/sep-2350.yaml b/src/seps/sep-2350.yaml new file mode 100644 index 00000000..486c768a --- /dev/null +++ b/src/seps/sep-2350.yaml @@ -0,0 +1,11 @@ +sep: 2350 +spec_url: https://modelcontextprotocol.io/specification/draft/basic/authorization#runtime-insufficient-scope-errors +requirements: + - check: sep-2350-scope-union-on-reauth + text: 'When re-authorizing, clients SHOULD include these scopes alongside any previously granted scopes to avoid losing permissions needed for other operations. / Clients SHOULD compute the union of previously requested scopes and newly challenged scopes when initiating re-authorization.' + url: https://modelcontextprotocol.io/specification/draft/basic/authorization#protected-resource-metadata-discovery-requirements + - check: sep-2350-server-single-challenge + text: 'Regardless of the approach chosen, servers SHOULD include all scopes required for the current operation in a single challenge.' + + - text: 'When responding with insufficient scope errors, servers SHOULD include the scopes needed to satisfy the current operation in the scope parameter, consistent with RFC 6750 Section 3.1.' + excluded: 'reword of pre-existing requirement (request->operation, +RFC6750 cite); no normative delta; harness already emits scope= in WWW-Authenticate' From 5c1b815cd1e03ee3daa435d5755bbc728dea5397 Mon Sep 17 00:00:00 2001 From: Anubhav Dhawan Date: Wed, 20 May 2026 01:40:52 +0530 Subject: [PATCH 02/13] feat: add client conformance tests for SEP-2575 (#270) * feat: merge latest request-metadata scenario stacked branch * test: implement optional capability conditional checks and simulated negotiation retry --- .../clients/typescript/everything-client.ts | 116 ++++++++ src/scenarios/client/request-metadata.test.ts | 236 +++++++++++++++ src/scenarios/client/request-metadata.ts | 272 ++++++++++++++++++ src/scenarios/index.ts | 2 + src/seps/sep-2575.yaml | 104 +++++++ 5 files changed, 730 insertions(+) create mode 100644 src/scenarios/client/request-metadata.test.ts create mode 100644 src/scenarios/client/request-metadata.ts create mode 100644 src/seps/sep-2575.yaml diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 48c107e1..650cbf60 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -87,6 +87,122 @@ async function runBasicClient(serverUrl: string): Promise { registerScenarios(['initialize', 'tools-call'], runBasicClient); +// ============================================================================ +// request-metadata scenario (SEP-2575) +// ============================================================================ + +async function runRequestMetadataClient(serverUrl: string): Promise { + logger.debug('Starting request-metadata client flow...'); + + const meta = { + 'io.modelcontextprotocol/clientInfo': { + name: 'conformance-test-client', + version: '1.0.0' + }, + 'io.modelcontextprotocol/clientCapabilities': { + tools: {}, + roots: {}, + sampling: {}, + elicitation: {} + } + }; + + let activeVersion = 'DRAFT-2026-v1'; + + const sendRequestWithNegotiation = async ( + method: string, + requestId: string | number, + params: any + ): Promise => { + const getPayload = (version: string) => ({ + jsonrpc: '2.0', + id: requestId, + method, + params: { + ...params, + _meta: { + ...params?._meta, + 'io.modelcontextprotocol/protocolVersion': version + } + } + }); + + const send = async (version: string) => { + return fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': version + }, + body: JSON.stringify(getPayload(version)) + }); + }; + + let response = await send(activeVersion); + if (response.status === 400) { + const clone = response.clone(); + try { + const errorResult = await clone.json(); + if (errorResult.error?.code === -32001) { + logger.debug( + 'Received UnsupportedProtocolVersionError, starting negotiation...' + ); + const serverSupported: string[] = + errorResult.error.data?.supported || []; + const clientSupported = ['DRAFT-2026-v1']; + const mutuallySupported = clientSupported.filter((v) => + serverSupported.includes(v) + ); + if (mutuallySupported.length > 0) { + activeVersion = mutuallySupported[0]; + logger.debug( + `Mutually supported version found: ${activeVersion}. Retrying...` + ); + response = await send(activeVersion); + } else { + logger.debug('No mutually supported version found. Aborting.'); + } + } + } catch (err) { + logger.debug('Failed to parse error response as JSON:', err); + } + } + + if (!response.ok) { + throw new Error(`${method} failed: ${response.status}`); + } + return response.json(); + }; + + // Call server/discover (optional for clients, but every POST still needs + // the header + _meta). + logger.debug('Calling server/discover...'); + const discoverResult = await sendRequestWithNegotiation( + 'server/discover', + 'discover-1', + { _meta: meta } + ); + logger.debug( + 'Successfully discovered server capabilities:', + JSON.stringify(discoverResult.result) + ); + + // Call tools/list with required inline _meta tags and header + logger.debug('Calling tools/list with inline _meta...'); + const toolsResult = await sendRequestWithNegotiation('tools/list', 2, { + _meta: meta + }); + logger.debug( + 'Successfully listed tools statelessly:', + JSON.stringify(toolsResult.result) + ); + + logger.debug('request-metadata client flow completed successfully'); +} + +// Register the scenario handler +registerScenario('request-metadata', runRequestMetadataClient); + // ============================================================================ // Auth scenarios - well-behaved client // ============================================================================ diff --git a/src/scenarios/client/request-metadata.test.ts b/src/scenarios/client/request-metadata.test.ts new file mode 100644 index 00000000..41ba4c42 --- /dev/null +++ b/src/scenarios/client/request-metadata.test.ts @@ -0,0 +1,236 @@ +import { describe, test, expect } from 'vitest'; +import { + runClientAgainstScenario, + InlineClientRunner +} from './auth/test_helpers/testClient'; +import { getHandler } from '../../../examples/clients/typescript/everything-client'; +import { getScenario } from '../index'; + +// A bad client that does not send _meta +async function badClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: {} // Missing _meta + }) + }); + return response.json(); +} + +const goodMeta = { + 'io.modelcontextprotocol/protocolVersion': 'DRAFT-2026-v1', + 'io.modelcontextprotocol/clientInfo': { name: 'test', version: '1.0' }, + 'io.modelcontextprotocol/clientCapabilities': {} +}; + +// A client that misses the HTTP header +async function missingHeaderClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, // Missing MCP-Protocol-Version header + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: goodMeta } + }) + }); + return response.json(); +} + +// A client whose header disagrees with _meta.protocolVersion +async function mismatchedHeaderClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': '2025-11-25' // != _meta.protocolVersion + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: goodMeta } + }) + }); + return response.json(); +} + +// A client that fails to negotiate/retry on a 400 response +async function nonRetryingClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': 'DRAFT-2026-v1' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { _meta: goodMeta } + }) + }); + return response.json(); +} + +// A client that has empty version intersection and terminates +async function incompatibleVersionClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': 'UNSUPPORTED-VERSION' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { + _meta: { + ...goodMeta, + 'io.modelcontextprotocol/protocolVersion': 'UNSUPPORTED-VERSION' + } + } + }) + }); + + if (response.status === 400) { + const body = await response.json(); + if (body.error?.code === -32001) { + return body; // Abort cleanly + } + } + return response.json(); +} + +// A client that sends invalid (non-object) capabilities +async function malformedCapabilitiesClient(serverUrl: string) { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'MCP-Protocol-Version': 'DRAFT-2026-v1' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list', + params: { + _meta: { + ...goodMeta, + 'io.modelcontextprotocol/clientCapabilities': { + roots: 'malformed-string', + sampling: {}, + elicitation: true + } + } + } + }) + }); + return response.json(); +} + +describe('request-metadata client scenario — positive test', () => { + test('everything-client passes request-metadata scenario with success status for optional capabilities', async () => { + const clientFn = getHandler('request-metadata'); + if (!clientFn) { + throw new Error('No handler registered for scenario: request-metadata'); + } + + const scenario = getScenario('request-metadata'); + if (!scenario) { + throw new Error('Scenario not found'); + } + + const runner = new InlineClientRunner(clientFn); + await runClientAgainstScenario(runner, 'request-metadata'); + + // Extract checks directly from the scenario instance + const checks = scenario.getChecks(); + + // 4-Line Bulk Assertion Loop + for (const check of checks) { + expect(check.status).not.toBe('FAILURE'); + expect(check.status).not.toBe('WARNING'); + } + + // Strategic Targeted Optional Assertions + expect( + checks.find((c) => c.id === 'sep-2575-client-declares-roots-capability') + ?.status + ).toBe('SUCCESS'); + expect( + checks.find( + (c) => c.id === 'sep-2575-client-declares-sampling-capability' + )?.status + ).toBe('SUCCESS'); + expect( + checks.find( + (c) => c.id === 'sep-2575-client-declares-elicitation-capability' + )?.status + ).toBe('SUCCESS'); + + // Assert version negotiation retry succeeded + expect( + checks.find((c) => c.id === 'sep-2575-client-retry-supported-version') + ?.status + ).toBe('SUCCESS'); + }); +}); + +describe('request-metadata client scenario — negative tests', () => { + test('client fails when omitting _meta', async () => { + const runner = new InlineClientRunner(badClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: [ + 'sep-2575-client-populates-meta', + 'sep-2575-http-client-sends-version-header' + ] + }); + }); + + test('client fails when missing version header', async () => { + const runner = new InlineClientRunner(missingHeaderClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: ['sep-2575-http-client-sends-version-header'] + }); + }); + + test('client fails when header disagrees with _meta', async () => { + const runner = new InlineClientRunner(mismatchedHeaderClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: ['sep-2575-http-version-header-matches-meta'] + }); + }); + + test('client fails retry check when it does not handle 400 rejection', async () => { + const runner = new InlineClientRunner(nonRetryingClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: ['sep-2575-client-retry-supported-version'] + }); + }); + + test('client aborts cleanly without hanging when negotiation has empty version intersection', async () => { + const runner = new InlineClientRunner(incompatibleVersionClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: ['sep-2575-client-retry-supported-version'] + }); + }); + + test('client triggers failures for malformed capabilities', async () => { + const runner = new InlineClientRunner(malformedCapabilitiesClient); + await runClientAgainstScenario(runner, 'request-metadata', { + expectedFailureSlugs: [ + 'sep-2575-client-declares-roots-capability', + 'sep-2575-client-declares-elicitation-capability' + ] + }); + }); +}); diff --git a/src/scenarios/client/request-metadata.ts b/src/scenarios/client/request-metadata.ts new file mode 100644 index 00000000..af1fde0d --- /dev/null +++ b/src/scenarios/client/request-metadata.ts @@ -0,0 +1,272 @@ +import http from 'http'; +import { + Scenario, + ScenarioUrls, + ConformanceCheck, + DRAFT_PROTOCOL_VERSION +} from '../../types'; + +export class RequestMetadataScenario implements Scenario { + name = 'request-metadata'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Per-request _meta and MCP-Protocol-Version header obligations (SEP-2575)'; + + private server: http.Server | null = null; + private checks: ConformanceCheck[] = []; + private hasSimulatedRejection = false; + + async start(): Promise { + this.hasSimulatedRejection = false; + this.checks = []; + return new Promise((resolve, reject) => { + this.server = http.createServer((req, res) => { + this.handleRequest(req, res); + }); + this.server.on('error', reject); + this.server.listen(0, () => { + const address = this.server!.address(); + if (address && typeof address === 'object') { + resolve({ serverUrl: `http://localhost:${address.port}` }); + } + }); + }); + } + + async stop(): Promise { + return new Promise((resolve) => { + if (this.server) { + this.server.close(() => { + resolve(); + }); + } else { + resolve(); + } + }); + } + + getChecks(): ConformanceCheck[] { + return this.checks; + } + + private addOrUpdateCheck(check: ConformanceCheck): void { + const index = this.checks.findIndex((c) => c.id === check.id); + if (index !== -1) { + this.checks[index] = check; + } else { + this.checks.push(check); + } + } + + private handleRequest( + req: http.IncomingMessage, + res: http.ServerResponse + ): void { + let body = ''; + req.on('data', (chunk) => { + body += chunk.toString(); + }); + req.on('end', () => { + const request = JSON.parse(body); + + // Extract version and headers + const meta = request.params?._meta; + const metaVersion = meta?.['io.modelcontextprotocol/protocolVersion']; + const headerVersion = req.headers['mcp-protocol-version']; + + // 1. "Every POST request to the MCP endpoint MUST include an + // MCP-Protocol-Version header." — unconditional, so this fires for + // server/discover too. + this.addOrUpdateCheck({ + id: 'sep-2575-http-client-sends-version-header', + name: 'ClientSendsVersionHeader', + description: 'Client sends MCP-Protocol-Version header on every POST', + status: headerVersion !== undefined ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' + } + ], + details: { method: request.method, headerVersion } + }); + + // 2. "Every client request MUST include the following + // io.modelcontextprotocol/* fields in _meta: protocolVersion, + // clientInfo, clientCapabilities." + const hasClientInfo = meta?.['io.modelcontextprotocol/clientInfo']; + const hasCapabilities = + meta?.['io.modelcontextprotocol/clientCapabilities']; + const metaIsValid = metaVersion && hasClientInfo && hasCapabilities; + + this.addOrUpdateCheck({ + id: 'sep-2575-client-populates-meta', + name: 'ClientPopulatesMeta', + description: + 'Client populates _meta on every request with all three required fields', + status: metaIsValid ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/index#meta' + } + ], + details: { method: request.method, meta } + }); + + // 3. "The header value MUST match the io.modelcontextprotocol/protocolVersion + // field carried in the request body's _meta." Only meaningful when both + // are present; absence is already covered by the two checks above. + if (headerVersion !== undefined && metaVersion !== undefined) { + this.addOrUpdateCheck({ + id: 'sep-2575-http-version-header-matches-meta', + name: 'ClientVersionHeaderMatchesMeta', + description: + 'MCP-Protocol-Version header matches _meta.protocolVersion', + status: headerVersion === metaVersion ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' + } + ], + details: { headerVersion, metaVersion } + }); + } + + // 4. Optional client capabilities conditional verification + const capabilities = meta?.['io.modelcontextprotocol/clientCapabilities']; + const checkOptionalCapability = ( + capabilityName: string, + checkId: string, + checkName: string + ) => { + let status: 'SUCCESS' | 'FAILURE' | 'SKIPPED' = 'SKIPPED'; + if (capabilities && capabilityName in capabilities) { + const val = capabilities[capabilityName]; + const isValidObject = + typeof val === 'object' && val !== null && !Array.isArray(val); + status = isValidObject ? 'SUCCESS' : 'FAILURE'; + } + this.addOrUpdateCheck({ + id: checkId, + name: checkName, + description: `Client declares valid ${capabilityName} capability if present`, + status, + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/index#capabilities' + } + ], + details: { capabilityValue: capabilities?.[capabilityName] } + }); + }; + + checkOptionalCapability( + 'roots', + 'sep-2575-client-declares-roots-capability', + 'ClientDeclaresRootsCapability' + ); + checkOptionalCapability( + 'sampling', + 'sep-2575-client-declares-sampling-capability', + 'ClientDeclaresSamplingCapability' + ); + checkOptionalCapability( + 'elicitation', + 'sep-2575-client-declares-elicitation-capability', + 'ClientDeclaresElicitationCapability' + ); + + // 5. Simulated Version Negotiation Retry Check + if (!this.hasSimulatedRejection) { + this.hasSimulatedRejection = true; + + this.addOrUpdateCheck({ + id: 'sep-2575-client-retry-supported-version', + name: 'ClientRetrySupportedVersion', + description: + 'Client retries with a supported version when first choice is rejected', + status: 'WARNING', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'SEP-2575', + url: 'https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header' + } + ], + details: { headerVersion } + }); + + res.writeHead(400, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id: request.id ?? null, + error: { + code: -32001, + message: 'Unsupported protocol version', + data: { + supported: [DRAFT_PROTOCOL_VERSION] + } + } + }) + ); + return; + } + + const retryCheck = this.checks.find( + (c) => c.id === 'sep-2575-client-retry-supported-version' + ); + if (retryCheck) { + if ( + headerVersion === DRAFT_PROTOCOL_VERSION && + metaVersion === DRAFT_PROTOCOL_VERSION + ) { + retryCheck.status = 'SUCCESS'; + } else { + retryCheck.status = 'WARNING'; + } + retryCheck.details = { + ...retryCheck.details, + retryHeaderVersion: headerVersion, + retryMetaVersion: metaVersion + }; + } + + // server/discover is optional for clients (spec: "Clients MAY call it"), + // so no check is emitted; we still respond so a client that does call it + // proceeds normally and exercises the per-request _meta/header checks above. + if (request.method === 'server/discover') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + result: { + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: {}, + serverInfo: { name: 'test', version: '1.0' } + } + }) + ); + return; + } + + // Return generic response to unblock client + let result: object = {}; + if (request.method === 'tools/list') { + result = { tools: [] }; + } else if (request.method === 'tools/call') { + result = { content: [] }; + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ jsonrpc: '2.0', id: request.id, result })); + }); + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index a46604fc..4b812d94 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -13,6 +13,7 @@ import { InitializeScenario } from './client/initialize'; import { ToolsCallScenario } from './client/tools_call'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; import { SSERetryScenario } from './client/sse-retry'; +import { RequestMetadataScenario } from './client/request-metadata'; // Import all new server test scenarios import { ServerInitializeScenario } from './server/lifecycle'; @@ -201,6 +202,7 @@ const scenariosList: Scenario[] = [ new ToolsCallScenario(), new ElicitationClientDefaultsScenario(), new SSERetryScenario(), + new RequestMetadataScenario(), ...authScenariosList, ...backcompatScenariosList, ...draftScenariosList, diff --git a/src/seps/sep-2575.yaml b/src/seps/sep-2575.yaml new file mode 100644 index 00000000..e48b1e26 --- /dev/null +++ b/src/seps/sep-2575.yaml @@ -0,0 +1,104 @@ +sep: 2575 +spec_url: https://modelcontextprotocol.io/specification/draft/basic/lifecycle +requirements: + - check: sep-2575-client-populates-meta + text: 'Every client request MUST include the following io.modelcontextprotocol/* fields in _meta: protocolVersion, clientInfo, clientCapabilities.' + url: https://modelcontextprotocol.io/specification/draft/basic/index#meta + - check: sep-2575-server-rejects-undeclared-capability + text: 'A server MUST NOT rely on capabilities the client has not declared. If processing a request requires a capability the client did not include in io.modelcontextprotocol/clientCapabilities, the server MUST return a MissingRequiredClientCapabilityError (-32003).' + url: https://modelcontextprotocol.io/specification/draft/basic/index#meta + - check: sep-2575-missing-capability-http-400 + text: 'On HTTP, the response status MUST be 400 Bad Request [for MissingRequiredClientCapabilityError].' + url: https://modelcontextprotocol.io/specification/draft/basic/index#meta + - check: sep-2575-server-tags-subscription-id + text: 'On notifications delivered via a subscriptions/listen stream, the server MUST include io.modelcontextprotocol/subscriptionId in _meta so the client can correlate the notification with the originating subscription request.' + url: https://modelcontextprotocol.io/specification/draft/basic/index#meta + - check: sep-2575-server-stateless-no-prior-context + text: 'A server MUST NOT treat connection or process identity as a proxy for conversation or session continuity. / Servers MUST NOT rely on prior requests over the same connection to establish context (e.g., capabilities, protocol version, client identity).' + - check: sep-2575-server-stateless-no-connection-reuse-required + text: 'Servers MUST NOT require that a client reuse the same connection to perform related operations.' + - check: sep-2575-server-unsupported-version-error + text: 'If the server does not implement the requested version (whether the version is unknown to the server, or is a known version the server has chosen not to support), it MUST respond with an UnsupportedProtocolVersionError listing the versions it does support.' + url: https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation + - check: sep-2575-client-retry-supported-version + text: 'The client SHOULD select a mutually supported version from the supported list and retry the request, or surface an error to the user if no compatible version exists.' + url: https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation + - check: sep-2575-server-implements-discover + text: 'Servers MUST implement server/discover.' + url: https://modelcontextprotocol.io/specification/draft/server/discover + - check: sep-2575-http-server-no-independent-requests-on-stream + text: 'The server MUST NOT send independent JSON-RPC requests on this stream. Server-to-client interactions are embedded as input requests inside an IncompleteResult.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#receiving-messages-1 + - check: sep-2575-http-server-disconnect-is-cancel + text: 'Closing the SSE response stream MUST be treated by the server as cancellation of that request.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1 + - check: sep-2575-http-server-stops-on-cancel + text: 'The server SHOULD stop work on the cancelled request as soon as practical and MUST NOT send any further messages for it [HTTP].' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1 + - check: sep-2575-http-client-sends-version-header + text: 'Every POST request to the MCP endpoint MUST include an MCP-Protocol-Version header.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header + - check: sep-2575-http-version-header-matches-meta + text: 'The header value MUST match the io.modelcontextprotocol/protocolVersion field carried in the request body _meta.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header + - check: sep-2575-http-server-header-mismatch-400 + text: 'If the values do not match, the server MUST reject the request with 400 Bad Request and a HeaderMismatch JSON-RPC error.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header + - check: sep-2575-http-server-unsupported-version-400 + text: 'If the server does not implement the requested protocol version, it MUST respond with 400 Bad Request and an UnsupportedProtocolVersionError listing its supported versions.' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header + - check: sep-2575-http-server-method-not-found-404 + text: 'If the server does not implement the requested RPC method, it MUST respond with 404 Not Found and a JSON-RPC error with code -32601 (Method not found).' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header + - check: sep-2575-server-honors-notification-filter + text: 'The server MUST NOT send notification types the client has not explicitly requested.' + url: https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#opening-a-stream + - check: sep-2575-server-sends-subscription-ack + text: 'The server MUST send notifications/subscriptions/acknowledged as the first message on the stream.' + url: https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#acknowledgment + - check: sep-2575-client-declares-elicitation-capability + text: 'Clients that support elicitation MUST declare the elicitation capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.' + url: https://modelcontextprotocol.io/specification/draft/client/elicitation#capabilities + - check: sep-2575-client-declares-roots-capability + text: 'Clients that support roots MUST declare the roots capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.' + url: https://modelcontextprotocol.io/specification/draft/client/roots#capabilities + - check: sep-2575-client-declares-sampling-capability + text: 'Clients that support sampling MUST declare the sampling capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.' + url: https://modelcontextprotocol.io/specification/draft/client/sampling#capabilities + - check: sep-2575-server-declares-prompts-in-discover + text: 'Servers that support prompts MUST declare the prompts capability in their DiscoverResult.' + url: https://modelcontextprotocol.io/specification/draft/server/prompts#capabilities + - check: sep-2575-server-sends-prompts-list-changed-on-subscription + text: '[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with promptsListChanged: true.' + url: https://modelcontextprotocol.io/specification/draft/server/prompts#list-changed-notification + - check: sep-2575-server-sends-tools-list-changed-on-subscription + text: '[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with toolsListChanged: true.' + url: https://modelcontextprotocol.io/specification/draft/server/tools#list-changed-notification + - check: sep-2575-server-no-log-without-loglevel + text: 'The server MUST NOT emit notifications/message for a request that does not include [io.modelcontextprotocol/logLevel in _meta].' + url: https://modelcontextprotocol.io/specification/draft/server/utilities/logging#per-request-log-level + + - text: 'State that needs to span multiple requests (e.g., long-running tasks, application-level handles) MUST be referenced by an explicit identifier the client passes on each request.' + excluded: 'architectural guidance, observable only via subscriptionId/task-id rows already listed' + - text: 'To distinguish notifications belonging to different concurrent subscriptions, clients MUST correlate notifications using the io.modelcontextprotocol/subscriptionId field carried in _meta.' + excluded: 'client-internal demux; not observable on the wire from the harness' + - text: 'The client SHOULD check the acknowledged filter against what it requested and handle any unsupported types gracefully.' + excluded: 'internal comparison; "gracefully" has no wire-observable definition' + - text: 'Because there is no per-request status code to drive fallback, a client that supports both eras SHOULD probe with server/discover first [stdio backward compatibility].' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/lifecycle#backward-compatibility-with-initialization-based-versions + - text: 'To cancel an in-flight request [on stdio], the client MUST send a notifications/cancelled notification referencing the request ID.' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation + - text: 'Servers SHOULD stop work on a cancelled request as soon as practical and MUST NOT send any further messages for it [stdio].' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation + - text: 'If the server process exits unexpectedly, the client SHOULD restart it.' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#unexpected-termination + - text: 'If the server returns UnsupportedProtocolVersionError, [the stdio client] SHOULD retry using one of the advertised supportedVersions rather than falling back to initialize.' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/transports#backward-compatibility + - text: 'On stdio, if the connection is terminated and then re-established, the client MUST re-send subscriptions/listen to re-establish its subscriptions.' + excluded: 'stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258' + url: https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#cancellation From 70f7ba00046b7eb9c9152e6439dea41b859514e4 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 21:48:31 +0100 Subject: [PATCH 03/13] fix(sep-2243): collapse check IDs to one-per-requirement (#287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sep-2243 scenario check IDs drifted from the requirement-traceability yaml in #259 — the code emitted one ID per test case while the yaml declares one per normative requirement, so the IDs no longer matched. Rename the emitted IDs so a check ID maps to a single MUST/SHOULD and is emitted once per case (the per-case detail moves to name/description), matching the repo's existing 'same id, vary status/message' convention: - mcp-method-header-* / mcp-name-header-* -> client-includes-standard-headers - reject-invalid-tool-* / keep-valid-tool -> client-reject-invalid-tool - server-accepts-{lower,upper}case-name -> header-name-case-insensitive - server reject status checks (mismatch/missing/case x method/name) -> server-reject-invalid-headers - their error-code variants -> server-reject-error-code Merge the yaml's server-reject-mismatch + server-reject-status into one server-reject-invalid-headers MUST (HTTP 400 on a header-validation failure); keep server-reject-error-code as the SHOULD. Base64/custom-header rejection checks keep their own param-validation IDs (out of scope here). Gates and the whitespace-acceptance check keep their scenario-matching names. No behavior change — only check IDs, descriptions, and the yaml. --- src/scenarios/client/http-custom-headers.ts | 4 +- .../client/http-standard-headers.test.ts | 13 ++++-- src/scenarios/client/http-standard-headers.ts | 8 ++-- src/scenarios/server/http-standard-headers.ts | 40 ++++++++++++++----- src/seps/sep-2243.yaml | 6 +-- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/src/scenarios/client/http-custom-headers.ts b/src/scenarios/client/http-custom-headers.ts index 1df5cbea..aa363a5c 100644 --- a/src/scenarios/client/http-custom-headers.ts +++ b/src/scenarios/client/http-custom-headers.ts @@ -626,7 +626,7 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { // Check that valid_tool WAS called — proves client kept valid tools const validToolCalled = this.calledTools.has('valid_tool'); this.checks.push({ - id: 'sep-2243-keep-valid-tool', + id: 'sep-2243-client-reject-invalid-tool', name: 'ClientKeepsValidTool', description: 'Client MUST keep valid tools while excluding invalid ones', status: validToolCalled ? 'SUCCESS' : 'FAILURE', @@ -654,7 +654,7 @@ export class HttpInvalidToolHeadersScenario extends BaseHttpScenario { for (const toolName of invalidTools) { const called = this.calledTools.has(toolName); this.checks.push({ - id: `sep-2243-reject-invalid-tool-${toolName.replace(/_/g, '-')}`, + id: 'sep-2243-client-reject-invalid-tool', name: `ClientRejectsInvalidTool_${toolName}`, description: `Client MUST NOT call tool '${toolName}' with invalid x-mcp-header`, status: called ? 'FAILURE' : 'SUCCESS', diff --git a/src/scenarios/client/http-standard-headers.test.ts b/src/scenarios/client/http-standard-headers.test.ts index 2d4c3038..8c5ff6ad 100644 --- a/src/scenarios/client/http-standard-headers.test.ts +++ b/src/scenarios/client/http-standard-headers.test.ts @@ -31,14 +31,19 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { }); } - it('FAILs sep-2243-mcp-method-header-initialize when Mcp-Method is missing', async () => { + // The coarse check id is emitted once per method/name case, so we narrow to + // the initialize Mcp-Method emission via its (case-specific) name. + const COARSE_ID = 'sep-2243-client-includes-standard-headers'; + const INIT_METHOD_NAME = 'ClientMcpMethodHeader_initialize'; + + it('FAILs the initialize Mcp-Method emission when Mcp-Method is missing', async () => { const scenario = new HttpStandardHeadersScenario(); const { serverUrl } = await scenario.start(); try { await postInitialize(serverUrl, {}); // no Mcp-Method header const checks = scenario.getChecks(); const check = checks.find( - (c) => c.id === 'sep-2243-mcp-method-header-initialize' + (c) => c.id === COARSE_ID && c.name === INIT_METHOD_NAME ); expect(check?.status).toBe('FAILURE'); } finally { @@ -46,14 +51,14 @@ describe('HttpStandardHeadersScenario (SEP-2243) — negative', () => { } }); - it('SUCCEEDs sep-2243-mcp-method-header-initialize when Mcp-Method matches', async () => { + it('SUCCEEDs the initialize Mcp-Method emission when Mcp-Method matches', async () => { const scenario = new HttpStandardHeadersScenario(); const { serverUrl } = await scenario.start(); try { await postInitialize(serverUrl, { 'Mcp-Method': 'initialize' }); const checks = scenario.getChecks(); const check = checks.find( - (c) => c.id === 'sep-2243-mcp-method-header-initialize' + (c) => c.id === COARSE_ID && c.name === INIT_METHOD_NAME ); expect(check?.status).toBe('SUCCESS'); } finally { diff --git a/src/scenarios/client/http-standard-headers.ts b/src/scenarios/client/http-standard-headers.ts index b3efd05e..fb527d00 100644 --- a/src/scenarios/client/http-standard-headers.ts +++ b/src/scenarios/client/http-standard-headers.ts @@ -53,7 +53,7 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { for (const method of expectedMethods) { if (!this.methodHeaderChecks.has(method)) { result.push({ - id: `sep-2243-mcp-method-header-${method.replace(/\//g, '-')}`, + id: 'sep-2243-client-includes-standard-headers', name: `ClientMcpMethodHeader_${method.replace(/\//g, '_')}`, description: `Client sends correct Mcp-Method header on ${method} request`, status: 'SKIPPED', @@ -68,7 +68,7 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { for (const method of expectedNameMethods) { if (!this.nameHeaderChecks.has(method)) { result.push({ - id: `sep-2243-mcp-name-header-${method.replace(/\//g, '-')}`, + id: 'sep-2243-client-includes-standard-headers', name: `ClientMcpNameHeader_${method.replace(/\//g, '_')}`, description: `Client sends correct Mcp-Name header on ${method} request`, status: 'SKIPPED', @@ -141,7 +141,7 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { this.methodHeaderChecks.set(method, errors.length === 0); this.checks.push({ - id: `sep-2243-mcp-method-header-${method.replace(/\//g, '-')}`, + id: 'sep-2243-client-includes-standard-headers', name: `ClientMcpMethodHeader_${method.replace(/\//g, '_')}`, description: `Client sends correct Mcp-Method header on ${method} request`, status: errors.length === 0 ? 'SUCCESS' : 'FAILURE', @@ -186,7 +186,7 @@ export class HttpStandardHeadersScenario extends BaseHttpScenario { this.nameHeaderChecks.set(method, errors.length === 0); this.checks.push({ - id: `sep-2243-mcp-name-header-${method.replace(/\//g, '-')}`, + id: 'sep-2243-client-includes-standard-headers', name: `ClientMcpNameHeader_${method.replace(/\//g, '_')}`, description: `Client sends correct Mcp-Name header on ${method} request`, status: errors.length === 0 ? 'SUCCESS' : 'FAILURE', diff --git a/src/scenarios/server/http-standard-headers.ts b/src/scenarios/server/http-standard-headers.ts index 06ca23bc..888d419a 100644 --- a/src/scenarios/server/http-standard-headers.ts +++ b/src/scenarios/server/http-standard-headers.ts @@ -53,6 +53,14 @@ const SPEC_REFERENCE_CUSTOM = { const HEADER_MISMATCH_ERROR_CODE = -32001; +// Coarse, requirement-level check IDs (SEP-2243) for STANDARD-header +// rejections. Every standard-header rejection case emits this same pair of IDs; +// the per-case name/description carry the detail of which case was exercised, +// matching the repo's "same id, vary status/message" convention. (Custom-header +// /Base64 rejections map to different requirements and keep their own IDs.) +const REJECT_STATUS_CHECK_ID = 'sep-2243-server-reject-invalid-headers'; +const REJECT_ERROR_CODE_CHECK_ID = 'sep-2243-server-reject-error-code'; + /** * Helper to send a raw HTTP POST request with custom headers. * Uses Node.js http.request to preserve exact header casing and values, @@ -119,9 +127,16 @@ async function sendRawRequest( * but -32001 is SHOULD for *standard* headers (and MUST for *custom* headers, * §Server Behavior for Custom Headers) — so a server returning 400 with a * different error code is compliant for standard headers and must not FAIL. + * + * The two emitted check IDs are supplied explicitly by the caller: standard- + * header callers pass the coarse REJECT_STATUS_CHECK_ID/REJECT_ERROR_CODE_CHECK_ID + * so all standard-header cases collapse onto one requirement, while custom-header + * /Base64 callers pass their own per-case ids. The per-case `name`/`description` + * distinguish which rejection case was exercised. */ function createRejectionChecks( - id: string, + statusId: string, + errorCodeId: string, name: string, description: string, response: { status: number; body: any }, @@ -141,7 +156,7 @@ function createRejectionChecks( return [ { - id, + id: statusId, name, description, status: statusOk ? 'SUCCESS' : 'FAILURE', @@ -153,7 +168,7 @@ function createRejectionChecks( details: fullDetails }, { - id: `${id}-error-code`, + id: errorCodeId, name: `${name}ErrorCode`, description: `${description} — uses JSON-RPC error code -32001 (HeaderMismatch)`, status: codeOk ? 'SUCCESS' : opts.errorCodeSeverity, @@ -397,7 +412,7 @@ export class HttpHeaderValidationScenario implements ClientScenario { baseHeaders, nextId, 'accept', - 'sep-2243-server-accepts-lowercase-header-name', + 'sep-2243-header-name-case-insensitive', 'ServerAcceptsLowercaseHeaderName', 'Server MUST accept lowercase header name (mcp-method)', { jsonrpc: '2.0', id: 0, method: 'tools/list' }, @@ -412,7 +427,7 @@ export class HttpHeaderValidationScenario implements ClientScenario { baseHeaders, nextId, 'accept', - 'sep-2243-server-accepts-uppercase-header-name', + 'sep-2243-header-name-case-insensitive', 'ServerAcceptsUppercaseHeaderName', 'Server MUST accept uppercase header name (MCP-METHOD)', { jsonrpc: '2.0', id: 0, method: 'tools/list' }, @@ -471,10 +486,13 @@ export class HttpHeaderValidationScenario implements ClientScenario { ...extraHeaders }); if (expectation === 'reject') { - // Standard-header rejection: 400 is MUST, -32001 is SHOULD. + // Standard-header rejection: 400 is MUST, -32001 is SHOULD. All + // standard-header rejection cases collapse onto the coarse requirement + // ids; checkId/checkName still distinguish the case via name/details. checks.push( ...createRejectionChecks( - checkId, + REJECT_STATUS_CHECK_ID, + REJECT_ERROR_CODE_CHECK_ID, checkName, description, response, @@ -840,10 +858,12 @@ export class HttpCustomHeaderServerValidationScenario implements ClientScenario ); } else { // Custom-header rejection: both 400 and -32001 are MUST per - // §Server Behavior for Custom Headers. + // §Server Behavior for Custom Headers. These map to param-validation + // requirements, so they keep their per-case ids (status + -error-code). checks.push( ...createRejectionChecks( checkId, + `${checkId}-error-code`, checkName, description, response, @@ -899,10 +919,12 @@ export class HttpCustomHeaderServerValidationScenario implements ClientScenario } ); - // Custom-header rejection: both 400 and -32001 are MUST. + // Custom-header rejection: both 400 and -32001 are MUST. Keeps its own + // per-case ids (status + -error-code). checks.push( ...createRejectionChecks( 'sep-2243-server-rejects-missing-custom-header', + 'sep-2243-server-rejects-missing-custom-header-error-code', 'ServerRejectsMissingCustomHeader', 'Server MUST reject request where custom header is omitted but value is present in body', response, diff --git a/src/seps/sep-2243.yaml b/src/seps/sep-2243.yaml index e981da06..bb7803f1 100644 --- a/src/seps/sep-2243.yaml +++ b/src/seps/sep-2243.yaml @@ -5,10 +5,8 @@ requirements: text: 'The client MUST include the standard MCP request headers on each POST request. These headers are REQUIRED for compliance.' - check: sep-2243-header-name-case-insensitive text: 'Clients and servers MUST use case-insensitive comparisons for header names.' - - check: sep-2243-server-reject-mismatch - text: 'Servers that process the request body MUST reject requests where the values specified in the headers do not match the corresponding values in the request body.' - - check: sep-2243-server-reject-status - text: 'When rejecting a request due to header validation failure, servers MUST return HTTP status 400 Bad Request.' + - check: sep-2243-server-reject-invalid-headers + text: 'Servers that process the request body MUST reject requests with mismatched or missing standard-header values, returning HTTP 400 Bad Request.' - check: sep-2243-server-reject-error-code text: 'When rejecting a request due to header validation failure, servers SHOULD include a JSON-RPC error response using error code -32001.' - check: sep-2243-client-supports-custom-headers From cfcecdba2cdde9e885043fd0d34abec9381f3320 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 22:08:06 +0100 Subject: [PATCH 04/13] feat: `sdk` subcommand to run local conformance against any SDK ref (#277) * feat: add sdk subcommand to run conformance against any SDK ref (#250) * Revise sdk runner: explicit --mode, KNOWN_SDKS-only config, v1/v2 entries Addresses review feedback on the sdk subcommand: - Require --mode (client|server) and remove "both". Each invocation now tests exactly one side with its own exit code; the old default ran client then server but combined exit codes with ||=, which skipped the server side entirely whenever the client run failed. - Resolve build/run config from KNOWN_SDKS + CLI flags only; drop the conformance.config.yaml loader (no SDK ships one yet). The Zod schema stays as the type for the built-in entries. - Split the typescript entry by major version: typescript-sdk (v2/main, pnpm install + build:all, expected-failures.yaml) and typescript-sdk-v1 (v1.x, npm ci + build, conformance-baseline.yml). An entry may set `repo` (real clone target for an alias) and `defaultRef` (branch used when no @ref is given). parseSdkSpec now leaves ref undefined when omitted so defaultRef can apply; a trailing @ is treated as no ref. - Key the clone cache by ref (/) so different refs of the same repo no longer share one checkout. - Bound the server readiness probe with a per-request AbortSignal timeout so a server that accepts the socket but never responds can't hang past the overall deadline. * fix(sdk-runner): resolve -o to absolute path; add --expected-failures override; replaceAll for safeName --- .gitignore | 1 + AGENTS.md | 1 + README.md | 44 +++++ src/index.ts | 4 + src/sdk-runner/checkout.ts | 126 +++++++++++++ src/sdk-runner/config.ts | 25 +++ src/sdk-runner/index.ts | 293 ++++++++++++++++++++++++++++++ src/sdk-runner/known-sdks.ts | 61 +++++++ src/sdk-runner/sdk-runner.test.ts | 87 +++++++++ vitest.config.ts | 2 +- 10 files changed, 643 insertions(+), 1 deletion(-) create mode 100644 src/sdk-runner/checkout.ts create mode 100644 src/sdk-runner/config.ts create mode 100644 src/sdk-runner/index.ts create mode 100644 src/sdk-runner/known-sdks.ts create mode 100644 src/sdk-runner/sdk-runner.test.ts diff --git a/.gitignore b/.gitignore index e9d9434a..2fdd7d0a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ .vscode/ .idea/ .claude/settings.local.json +.sdk-under-test/ diff --git a/AGENTS.md b/AGENTS.md index fc864ba5..c91484eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ Keep scenarios separate when they're genuinely independent features or when they - **Same `id` for SUCCESS and FAIL.** A check should use one slug and flip `status` + `errorMessage`, not branch into `foo-success` vs `foo-failure` slugs. - **Optimize for Ctrl+F on the slug.** Repetitive check blocks are fine — easier to find the failing one than to unwind a clever helper. - Reuse `ConformanceCheck` and other types from `src/types.ts` rather than defining parallel shapes. +- **Don't reimplement the runner.** New subcommands that need to "select scenarios → run them → print summary → compute exit code" must go through the existing `client` / `server` commands (subprocess via `process.execPath` like `tier-check` and `sdk` do) or call shared helpers — never a parallel suite-map / summary loop. - Include `specReferences` pointing to the relevant spec section. - **Severity follows the spec keyword:** MUST / MUST NOT → `FAILURE`; SHOULD / SHOULD NOT → `WARNING`. (CI treats WARNING as a failure, so Tier-1 SDKs still need to satisfy SHOULDs — see #245.) diff --git a/README.md b/README.md index b2b5f0e4..333bfcb9 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,50 @@ Run `npx @modelcontextprotocol/conformance list --server` to see all available s - **resources-\*** - Resource management scenarios - **prompts-\*** - Prompt management scenarios +## Running Against an SDK at a Specific Ref + +The `sdk` subcommand clones an SDK repository at a given ref, builds it, and runs the **local** conformance build against it. This is the inner-loop tool for scenario authors and the basis for cross-SDK CI. Examples below use `npm start --` so they run from source — no `npm run build` between edits. + +`--mode client` or `--mode server` is required — each invocation tests exactly one side, so client and server are run (and pass/fail) independently. + +```bash +# Run the client conformance suite against typescript-sdk @main (v2) +npm start -- sdk typescript-sdk --mode client + +# Run the server conformance suite (separate invocation) +npm start -- sdk typescript-sdk --mode server + +# A specific main-line SHA or branch (v2 monorepo) +npm start -- sdk typescript-sdk@abc123f --mode client +npm start -- sdk typescript-sdk@some-branch --mode server + +# The published v1.x line — separate entry (npm build), defaults to the v1.x branch +npm start -- sdk typescript-sdk-v1 --mode client +npm start -- sdk typescript-sdk-v1@v1.29.0 --mode server + +# Use an existing local checkout (no clone, no fetch) +npm start -- sdk --path ../typescript-sdk --skip-build --mode client + +# Narrow to one scenario / suite +npm start -- sdk --path ../typescript-sdk --mode server --scenario server-initialize +npm start -- sdk typescript-sdk --mode client --suite auth +``` + +Build/run commands for each official SDK are looked up by name from [`src/sdk-runner/known-sdks.ts`](src/sdk-runner/known-sdks.ts) — no config file is required in the SDK repo. Resolution order is **CLI flag > built-in entry**, so any field can be overridden on the command line for refs that diverge from the built-in. + +An SDK can have more than one entry when its layout differs across major versions — e.g. `typescript-sdk` (v2, the `main` monorepo) and `typescript-sdk-v1` (the published npm v1.x line). An entry may set `defaultRef` (the branch used when you don't pass `@`) and `repo` (the real clone target when the entry name is an alias). Overriding for a one-off ref: + +```bash +npm start -- sdk owner/go-sdk@some-branch \ + --mode client \ + --build-cmd 'go build -tags mcp_go_client_oauth -o ./.conformance-client ./conformance/everything-client' \ + --client-cmd './.conformance-client' +``` + +To add a new SDK to the matrix, add an entry to `KNOWN_SDKS`. + +Clones are cached under `.sdk-under-test/` and reused (fetched) on subsequent runs. + ## SDK Tier Assessment The `tier-check` subcommand evaluates an MCP SDK repository against [SEP-1730](https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1730) (the SDK Tiering System): diff --git a/src/index.ts b/src/index.ts index 013f44c4..a3431cd3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -46,6 +46,7 @@ import { } from './expected-failures'; import { createTierCheckCommand } from './tier-check'; import { createNewSepCommand } from './new-sep'; +import { createSdkCommand } from './sdk-runner'; import packageJson from '../package.json'; // Note on naming: `command` refers to which CLI command is calling this. @@ -544,6 +545,9 @@ program.addCommand(createTierCheckCommand()); // New SEP scaffolding command program.addCommand(createNewSepCommand()); +// SDK command - run local conformance against an SDK at a specific ref +program.addCommand(createSdkCommand()); + // List scenarios command program .command('list') diff --git a/src/sdk-runner/checkout.ts b/src/sdk-runner/checkout.ts new file mode 100644 index 00000000..fb91e07a --- /dev/null +++ b/src/sdk-runner/checkout.ts @@ -0,0 +1,126 @@ +import { spawn } from 'child_process'; +import { promises as fs } from 'fs'; +import path from 'path'; + +export interface SdkSpec { + name: string; + ref: string; +} + +/** + * A parsed `[@]` argument. `ref` is left undefined when the user + * omits `@` so the caller can fall back to a per-SDK default branch + * (KNOWN_SDKS `defaultRef`) before settling on `main`. + */ +export interface ParsedSdkSpec { + name: string; + ref?: string; +} + +const DEFAULT_ORG = 'modelcontextprotocol'; + +export function parseSdkSpec(spec: string): ParsedSdkSpec { + const at = spec.lastIndexOf('@'); + if (at <= 0) { + return { name: spec }; + } + // A trailing `@` (empty ref) is treated as "no ref given" so the caller's + // defaultRef/main fallback applies, rather than checking out the empty ref. + const ref = spec.slice(at + 1); + return ref ? { name: spec.slice(0, at), ref } : { name: spec.slice(0, at) }; +} + +function repoUrl(name: string): string { + if (name.includes('/')) { + return `https://github.com/${name}.git`; + } + return `https://github.com/${DEFAULT_ORG}/${name}.git`; +} + +async function git( + args: string[], + cwd: string +): Promise<{ stdout: string; stderr: string }> { + const cmd = 'git'; + return new Promise((resolve, reject) => { + const child = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d.toString())); + child.stderr.on('data', (d) => (stderr += d.toString())); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject( + new Error( + `${cmd} ${args.join(' ')} exited with ${code}\n${stderr || stdout}` + ) + ); + } + }); + }); +} + +async function dirExists(dir: string): Promise { + try { + const stat = await fs.stat(dir); + return stat.isDirectory(); + } catch { + return false; + } +} + +/** + * Ensure an SDK is checked out at the requested ref under cacheDir. + * Clones on first use; on subsequent calls fetches and resets to the ref. + * Returns the absolute path to the checkout. + */ +export async function ensureCheckout( + spec: SdkSpec, + cacheDir: string +): Promise { + const safeName = spec.name.replace(/\//g, '__'); + // Key the checkout by ref as well, so different refs of the same repo (e.g. + // the typescript-sdk `main` and typescript-sdk-v1 `v1.x` entries) get their + // own directory instead of thrashing one checkout between refs/build systems. + const safeRef = spec.ref.replace(/[^a-zA-Z0-9._-]/g, '_'); + const dir = path.resolve(cacheDir, safeName, safeRef); + await fs.mkdir(path.dirname(dir), { recursive: true }); + + if (await dirExists(path.join(dir, '.git'))) { + console.error(`[sdk] Fetching ${spec.name} (cached at ${dir})`); + await git(['fetch', '--tags', 'origin'], dir); + } else { + console.error(`[sdk] Cloning ${repoUrl(spec.name)} -> ${dir}`); + await git(['clone', repoUrl(spec.name), dir], path.dirname(dir)); + } + + // Try the ref as a remote branch first, then fall back to a local-resolvable + // ref (tag or SHA). + const candidates = [`origin/${spec.ref}`, spec.ref]; + let resolved: string | undefined; + for (const candidate of candidates) { + try { + await git(['rev-parse', '--verify', `${candidate}^{commit}`], dir); + resolved = candidate; + break; + } catch { + // rev-parse failure means this candidate doesn't exist; try the next form + } + } + if (!resolved) { + throw new Error( + `Ref '${spec.ref}' not found in ${spec.name} (tried ${candidates.join(', ')})` + ); + } + + console.error(`[sdk] Checking out ${spec.name}@${spec.ref} (${resolved})`); + await git(['checkout', '--detach', resolved], dir); + + const { stdout } = await git(['rev-parse', '--short', 'HEAD'], dir); + console.error(`[sdk] HEAD is ${stdout.trim()}`); + + return dir; +} diff --git a/src/sdk-runner/config.ts b/src/sdk-runner/config.ts new file mode 100644 index 00000000..0945b996 --- /dev/null +++ b/src/sdk-runner/config.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +export const SdkConfigSchema = z.object({ + // Clone this repo instead of the KNOWN_SDKS key — lets an alias entry + // (e.g. typescript-sdk-v1) point at the real repo (typescript-sdk). + repo: z.string().optional(), + // Ref to check out when the SDK is named with no @ref (the "default branch"). + defaultRef: z.string().optional(), + build: z.string().optional(), + client: z + .object({ + command: z.string() + }) + .optional(), + server: z + .object({ + command: z.string(), + url: z.string().url(), + readyTimeoutMs: z.number().int().positive().optional() + }) + .optional(), + expectedFailures: z.string().optional() +}); + +export type SdkConfig = z.infer; diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts new file mode 100644 index 00000000..932c51e5 --- /dev/null +++ b/src/sdk-runner/index.ts @@ -0,0 +1,293 @@ +import { spawn, ChildProcess } from 'child_process'; +import path from 'path'; +import { Command, Option } from 'commander'; +import { SdkConfig } from './config'; +import { parseSdkSpec, ensureCheckout } from './checkout'; +import { lookupBuiltinConfig, knownSdkNames } from './known-sdks'; + +type Mode = 'client' | 'server'; + +function execShell(command: string, cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, { shell: true, cwd, stdio: 'inherit' }); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) resolve(); + else reject(new Error(`Command failed (exit ${code}): ${command}`)); + }); + }); +} + +/** + * Re-invoke this CLI as a subprocess so scenario selection / reporting stay in + * one place (same approach tier-check uses). Preserves execArgv so tsx/loader + * hooks carry over when running from source. + */ +function selfInvoke(args: string[], cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [...process.execArgv, process.argv[1], ...args], + { cwd, stdio: 'inherit' } + ); + child.on('error', reject); + child.on('close', (code) => resolve(code ?? 1)); + }); +} + +async function waitForReady(url: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + // Per-probe timeout: a server that accepts the socket but never responds must + // not block past the overall deadline (fetch has no timeout of its own). + const probeTimeoutMs = 2000; + let lastErr: unknown; + while (Date.now() < deadline) { + try { + await fetch(url, { + method: 'GET', + signal: AbortSignal.timeout(probeTimeoutMs) + }); + return; + } catch (err) { + lastErr = err; + await new Promise((r) => setTimeout(r, 250)); + } + } + throw new Error( + `Server at ${url} did not become ready within ${timeoutMs}ms: ${lastErr}` + ); +} + +async function withManagedServer( + command: string, + cwd: string, + url: string, + readyTimeoutMs: number, + fn: () => Promise +): Promise { + console.error(`[sdk] Starting server: ${command}`); + const child: ChildProcess = spawn(command, { + shell: true, + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32' + }); + + let stderr = ''; + child.stdout?.on('data', (d) => process.stderr.write(`[server] ${d}`)); + child.stderr?.on('data', (d) => { + stderr += d.toString(); + process.stderr.write(`[server] ${d}`); + }); + + let stopping = false; + const exited = new Promise((_, reject) => { + child.on('exit', (code) => { + if (stopping) return; + reject( + new Error( + `Server exited with code ${code} before tests completed\n${stderr}` + ) + ); + }); + child.on('error', reject); + }); + exited.catch(() => {}); + + try { + await Promise.race([waitForReady(url, readyTimeoutMs), exited]); + console.error(`[sdk] Server ready at ${url}`); + return await Promise.race([fn(), exited]); + } finally { + stopping = true; + console.error(`[sdk] Stopping server`); + if (process.platform !== 'win32' && child.pid) { + try { + process.kill(-child.pid, 'SIGTERM'); + } catch { + child.kill('SIGTERM'); + } + } else { + child.kill('SIGTERM'); + } + } +} + +function passThrough(options: { + scenario?: string; + suite?: string; + timeout?: string; + verbose?: boolean; + output?: string; +}): string[] { + const args: string[] = []; + if (options.scenario) args.push('--scenario', options.scenario); + else if (options.suite) args.push('--suite', options.suite); + if (options.timeout) args.push('--timeout', options.timeout); + if (options.verbose) args.push('--verbose'); + if (options.output) args.push('-o', options.output); + return args; +} + +export function createSdkCommand(): Command { + return new Command('sdk') + .description( + 'Run the local conformance build against an SDK checked out at a specific ref' + ) + .argument( + '[sdk]', + 'SDK to test as [@], e.g. typescript-sdk@main. Name may be owner/repo.' + ) + .option( + '--path ', + 'Use an existing local SDK checkout instead of cloning' + ) + .option( + '--cache-dir ', + 'Directory for cached SDK clones', + '.sdk-under-test' + ) + .addOption( + new Option( + '--mode ', + 'Which side to test (required): client or server' + ).choices(['client', 'server']) + ) + .option('--scenario ', 'Run a single scenario (passed through)') + .option('--suite ', 'Run a suite (passed through)') + .option('--skip-build', 'Skip the SDK build step (reuse prior build)') + .option('--build-cmd ', 'Override the build command from config') + .option('--client-cmd ', 'Override the client command from config') + .option('--server-cmd ', 'Override the server command from config') + .option('--server-url ', 'Override the server URL from config') + .option( + '--expected-failures ', + 'Override the expected-failures baseline file from config' + ) + .option('--timeout ', 'Per-scenario client timeout (passed through)') + .option('-o, --output ', 'Output directory (passed through)') + .option('--verbose', 'Verbose output (passed through)') + .action(async (sdkArg: string | undefined, options) => { + try { + const mode = options.mode as Mode | undefined; + if (!mode) { + throw new Error(`--mode is required (client | server)`); + } + if (!sdkArg && !options.path) { + throw new Error( + `Provide an SDK spec (e.g. typescript-sdk@main) or --path` + ); + } + + const spec = sdkArg ? parseSdkSpec(sdkArg) : undefined; + const sdkName = + spec?.name ?? path.basename(path.resolve(options.path!)); + + // Resolution: CLI flag > built-in entry (KNOWN_SDKS). + const builtinConfig: SdkConfig = lookupBuiltinConfig(sdkName) ?? {}; + + // The built-in entry may be an alias (e.g. typescript-sdk-v1): honor its + // `repo` (real clone target) and `defaultRef` (branch when no @ref given). + const dir = options.path + ? path.resolve(options.path) + : await ensureCheckout( + { + name: builtinConfig.repo ?? spec!.name, + ref: spec!.ref ?? builtinConfig.defaultRef ?? 'main' + }, + options.cacheDir + ); + const buildCmd: string | undefined = + options.buildCmd ?? builtinConfig.build; + const clientCmd: string | undefined = + options.clientCmd ?? builtinConfig.client?.command; + const serverCmd: string | undefined = + options.serverCmd ?? builtinConfig.server?.command; + const serverUrl: string | undefined = + options.serverUrl ?? builtinConfig.server?.url; + // CLI override resolves relative to the user's invocation cwd; the + // built-in default resolves relative to the SDK checkout. + const expectedFailures = options.expectedFailures + ? path.resolve(options.expectedFailures) + : builtinConfig.expectedFailures + ? path.resolve(dir, builtinConfig.expectedFailures) + : undefined; + // Resolve -o to an absolute path so it lands where the user expects, + // not relative to the SDK checkout (selfInvoke runs with cwd = dir). + const output = options.output + ? path.resolve(options.output) + : undefined; + + if (buildCmd && !options.skipBuild) { + console.error(`[sdk] Building: ${buildCmd}`); + await execShell(buildCmd, dir); + } else if (!buildCmd) { + console.error( + `[sdk] No build command in config; assuming SDK is already built` + ); + } + + let exitCode: number; + + if (mode === 'client') { + if (!clientCmd) { + throw new Error( + `No client command for '${sdkName}'. Pass --client-cmd, or add it to KNOWN_SDKS in src/sdk-runner/known-sdks.ts (known: ${knownSdkNames().join(', ')}).` + ); + } + const args = [ + 'client', + '--command', + clientCmd, + ...passThrough({ + scenario: options.scenario, + suite: options.suite ?? 'all', + timeout: options.timeout, + verbose: options.verbose, + output + }) + ]; + if (expectedFailures) + args.push('--expected-failures', expectedFailures); + console.error(`\n[sdk] conformance ${args.join(' ')}\n`); + exitCode = await selfInvoke(args, dir); + } else { + if (!serverCmd || !serverUrl) { + throw new Error( + `No server command/url for '${sdkName}'. Pass --server-cmd / --server-url, or add it to KNOWN_SDKS in src/sdk-runner/known-sdks.ts (known: ${knownSdkNames().join(', ')}).` + ); + } + const args = [ + 'server', + '--url', + serverUrl, + ...passThrough({ + scenario: options.scenario, + suite: options.suite, + verbose: options.verbose, + output + }) + ]; + if (expectedFailures) + args.push('--expected-failures', expectedFailures); + exitCode = await withManagedServer( + serverCmd, + dir, + serverUrl, + builtinConfig.server?.readyTimeoutMs ?? 15000, + async () => { + console.error(`\n[sdk] conformance ${args.join(' ')}\n`); + return selfInvoke(args, dir); + } + ); + } + + process.exit(exitCode); + } catch (error) { + console.error( + `[sdk] ${error instanceof Error ? error.message : String(error)}` + ); + process.exit(1); + } + }); +} diff --git a/src/sdk-runner/known-sdks.ts b/src/sdk-runner/known-sdks.ts new file mode 100644 index 00000000..b6550df1 --- /dev/null +++ b/src/sdk-runner/known-sdks.ts @@ -0,0 +1,61 @@ +import type { SdkConfig } from './config'; + +/** + * Built-in conformance configs for official SDKs, keyed by repo name. + * + * These live here (not in the SDK repos) so adding an SDK to the matrix + * doesn't require a coordinated cross-repo PR. Any field can be overridden + * per-invocation via the CLI flags (--build-cmd / --client-cmd / etc.). + */ +export const KNOWN_SDKS: Record = { + // v2 — the monorepo on `main` (pnpm). Default ref is `main`. + 'typescript-sdk': { + build: 'pnpm install && pnpm run build:all', + client: { + command: 'npx tsx test/conformance/src/everythingClient.ts' + }, + server: { + command: 'npx tsx test/conformance/src/everythingServer.ts', + url: 'http://localhost:3000/mcp' + }, + expectedFailures: 'test/conformance/expected-failures.yaml' + }, + // v1.x — the published npm line. Same fixtures as v2; differs only in the + // build (npm, not pnpm) and the baseline filename. Clones the typescript-sdk + // repo, defaulting to the `v1.x` branch. + 'typescript-sdk-v1': { + repo: 'typescript-sdk', + defaultRef: 'v1.x', + build: 'npm ci && npm run build', + client: { + command: 'npx tsx test/conformance/src/everythingClient.ts' + }, + server: { + command: 'npx tsx test/conformance/src/everythingServer.ts', + url: 'http://localhost:3000/mcp' + }, + expectedFailures: 'test/conformance/conformance-baseline.yml' + }, + 'go-sdk': { + build: 'go build -o ./.conformance-server ./examples/server/conformance', + // Upstream go-sdk has no client conformance fixture yet (see go-sdk#859). + server: { + command: './.conformance-server -http=:3000', + url: 'http://localhost:3000' + } + } +}; + +/** + * Look up a built-in config by SDK name. Accepts bare names (typescript-sdk), + * owner/repo (modelcontextprotocol/typescript-sdk), or a checkout path + * basename — only the final path segment is used as the key. + */ +export function lookupBuiltinConfig(name: string): SdkConfig | null { + const key = name.split('/').pop() ?? name; + return KNOWN_SDKS[key] ?? null; +} + +export function knownSdkNames(): string[] { + return Object.keys(KNOWN_SDKS); +} diff --git a/src/sdk-runner/sdk-runner.test.ts b/src/sdk-runner/sdk-runner.test.ts new file mode 100644 index 00000000..c58e9931 --- /dev/null +++ b/src/sdk-runner/sdk-runner.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import { parseSdkSpec } from './checkout'; +import { SdkConfigSchema } from './config'; +import { lookupBuiltinConfig, KNOWN_SDKS } from './known-sdks'; + +describe('parseSdkSpec', () => { + it('leaves ref undefined when omitted (resolved later via defaultRef/main)', () => { + expect(parseSdkSpec('typescript-sdk')).toEqual({ + name: 'typescript-sdk' + }); + }); + + it('splits name@ref', () => { + expect(parseSdkSpec('typescript-sdk@v1.29.0')).toEqual({ + name: 'typescript-sdk', + ref: 'v1.29.0' + }); + }); + + it('handles owner/repo@ref', () => { + expect(parseSdkSpec('someorg/some-sdk@abc123')).toEqual({ + name: 'someorg/some-sdk', + ref: 'abc123' + }); + }); + + it('treats leading @ as part of the name', () => { + expect(parseSdkSpec('@scope/pkg')).toEqual({ + name: '@scope/pkg' + }); + }); + + it('treats a trailing @ as no ref (falls through to defaultRef/main)', () => { + expect(parseSdkSpec('typescript-sdk@')).toEqual({ name: 'typescript-sdk' }); + }); +}); + +describe('SdkConfigSchema', () => { + it('accepts a minimal client-only config', () => { + const cfg = SdkConfigSchema.parse({ + client: { command: 'tsx fixture.ts' } + }); + expect(cfg.client?.command).toBe('tsx fixture.ts'); + expect(cfg.server).toBeUndefined(); + }); + + it('rejects server config without a url', () => { + expect(() => + SdkConfigSchema.parse({ server: { command: 'tsx server.ts' } }) + ).toThrow(); + }); +}); + +describe('lookupBuiltinConfig', () => { + it('finds an SDK by bare name', () => { + expect(lookupBuiltinConfig('typescript-sdk')?.client?.command).toBeTruthy(); + }); + + it('strips owner/ prefix and path segments', () => { + expect(lookupBuiltinConfig('modelcontextprotocol/typescript-sdk')).toBe( + KNOWN_SDKS['typescript-sdk'] + ); + expect(lookupBuiltinConfig('/some/path/to/go-sdk')).toBe( + KNOWN_SDKS['go-sdk'] + ); + }); + + it('returns null for unknown SDKs', () => { + expect(lookupBuiltinConfig('rust-sdk')).toBeNull(); + }); + + it('exposes the typescript-sdk-v1 alias with repo + defaultRef', () => { + const v1 = lookupBuiltinConfig('typescript-sdk-v1'); + expect(v1?.repo).toBe('typescript-sdk'); + expect(v1?.defaultRef).toBe('v1.x'); + }); + + it('bare typescript-sdk (v2) has no defaultRef', () => { + expect(lookupBuiltinConfig('typescript-sdk')?.defaultRef).toBeUndefined(); + }); + + it('every built-in entry validates against SdkConfigSchema', () => { + for (const [name, cfg] of Object.entries(KNOWN_SDKS)) { + expect(() => SdkConfigSchema.parse(cfg), name).not.toThrow(); + } + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 93242b59..b0f36f97 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,7 +5,7 @@ export default defineConfig({ globals: true, environment: 'node', include: ['**/*.test.ts'], - exclude: ['**/node_modules/**', 'dist'], + exclude: ['**/node_modules/**', 'dist', '.sdk-under-test'], // Run test files sequentially to avoid port conflicts fileParallelism: false, // Increase timeout for server tests in CI From 442ba3b1430651b32d38f88b947fa27bc9dd583e Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 22:22:25 +0100 Subject: [PATCH 05/13] feat: add SEP conformance traceability manifest (#288) squashed; see PR #288 body --- .github/workflows/traceability.yml | 123 +++++++++ .prettierignore | 5 + AGENTS.md | 22 ++ package.json | 1 + src/index.ts | 4 + src/seps/traceability.json | 409 +++++++++++++++++++++++++++++ src/traceability/index.test.ts | 124 +++++++++ src/traceability/index.ts | 316 ++++++++++++++++++++++ src/traceability/types.ts | 79 ++++++ 9 files changed, 1083 insertions(+) create mode 100644 .github/workflows/traceability.yml create mode 100644 .prettierignore create mode 100644 src/seps/traceability.json create mode 100644 src/traceability/index.test.ts create mode 100644 src/traceability/index.ts create mode 100644 src/traceability/types.ts diff --git a/.github/workflows/traceability.yml b/.github/workflows/traceability.yml new file mode 100644 index 00000000..44c85d9f --- /dev/null +++ b/.github/workflows/traceability.yml @@ -0,0 +1,123 @@ +name: Refresh SEP traceability manifest + +# Regenerates src/seps/traceability.json by running the conformance suite against +# the reference SDK and recording which check IDs were emitted, then opens a PR +# with the diff. NOT a PR gate — runs on demand / on a schedule and proposes an +# update for review. plan.modelcontextprotocol.io reads the committed file from +# main. +# +# Depends on the `conformance sdk` subcommand (#277), which clones+builds the SDK +# and runs the client+server suites. The `run` job executes third-party SDK code, +# so it has NO repo write token (read-only perms, persist-credentials: false) and +# only uploads results as an artifact; the separate `propose` job holds the +# write/PR permissions and never executes SDK code. + +on: + workflow_dispatch: + inputs: + sdk: + description: 'SDK ref to run against (e.g. typescript-sdk@)' + default: 'typescript-sdk@main' + schedule: + - cron: '0 6 * * 1' # Weekly, Monday 06:00 UTC. + +concurrency: + group: traceability-refresh + cancel-in-progress: true + +jobs: + run: + runs-on: ubuntu-latest + permissions: + contents: read + env: + SDK_REF: ${{ inputs.sdk || 'typescript-sdk@main' }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false # no git token while SDK code runs + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - run: npm ci + - run: npm run build + + - name: Run conformance suites against the reference SDK + # `sdk` requires --mode client|server; run both into the same results dir + # (the second reuses the cached checkout + build via --skip-build). + run: | + node dist/index.js sdk "$SDK_REF" --mode client --suite all -o results + node dist/index.js sdk "$SDK_REF" --mode server --suite all --skip-build -o results + + - name: Fail if no results were produced + run: | + if [ -z "$(find results -name checks.json -print -quit 2>/dev/null)" ]; then + echo "No checks.json produced — the suite run failed; not proposing a manifest." + exit 1 + fi + + - uses: actions/upload-artifact@v4 + with: + name: conformance-results + path: results + retention-days: 7 + + propose: + needs: run + runs-on: ubuntu-latest + # Requires the repo/org setting "Allow GitHub Actions to create and approve + # pull requests" to be enabled, otherwise `gh pr create` fails. + permissions: + contents: write + pull-requests: write + env: + SDK_REF: ${{ inputs.sdk || 'typescript-sdk@main' }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npm run build + + - uses: actions/download-artifact@v4 + with: + name: conformance-results + path: results + + - name: Regenerate manifest + run: | + set -euo pipefail + # Record the resolved sha (stable per SDK commit) so the manifest's + # `source` only changes when the SDK actually advances — no per-run noise. + ref="${SDK_REF#*@}" + sha="$(git ls-remote https://github.com/modelcontextprotocol/typescript-sdk.git "$ref" | cut -f1)" + node dist/index.js traceability --results results \ + --source "typescript-sdk@${sha:0:12}" + + - name: Open/update the rolling refresh PR + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if git diff --quiet -- src/seps/traceability.json; then + echo "traceability.json unchanged" + exit 0 + fi + # One rolling branch/PR, force-updated each run, so the schedule does + # not accrue a new PR every week. + branch="traceability-refresh" + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git checkout -B "$branch" + git add src/seps/traceability.json + git commit -m "chore: refresh SEP traceability manifest ($SDK_REF)" + git push --force origin "$branch" + gh pr view "$branch" >/dev/null 2>&1 || gh pr create \ + --head "$branch" \ + --title 'chore: refresh SEP traceability manifest' \ + --body 'Automated refresh from a conformance run against the reference SDK. Review the coverage diff before merging.' diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..43399fcc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,5 @@ +# Generated by `conformance traceability` — formatting is owned by the +# generator (deterministic JSON.stringify), not Prettier. Without this, the +# repo's `prettier --check .` would reformat the file and fight the generator's +# output (and the refresh workflow's `git diff` check). +src/seps/traceability.json diff --git a/AGENTS.md b/AGENTS.md index c91484eb..5e915fff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,6 +82,27 @@ npx @modelcontextprotocol/conformance new-sep The command looks up PR #`` in `modelcontextprotocol/modelcontextprotocol` (SEP numbers are PR numbers), derives `spec_url` from the `docs/specification/draft/*.mdx` file it changes, and writes `src/seps/sep-.yaml` with TODO `requirements[]` rows. Use `--spec-path` or `--spec-url` to skip the lookup. The `new-sep` Claude Code skill drives the same flow end-to-end, parses the spec diff, and fills in the requirement rows. +### Traceability manifest + +`src/seps/traceability.json` is a generated map of, per SEP, which declared `check:` IDs are actually emitted when the conformance suite runs against the reference SDK. It is consumed by plan.modelcontextprotocol.io to track SEP-2484 progress. + +The emitted check IDs come from a real suite run (not a source scan), so dynamic (template-literal) IDs resolve to their concrete values. Generate the manifest from a results directory: + +```sh +# 1. Run the suite against the reference SDK, collecting checks.json files: +node dist/index.js client --command '' --suite all -o results +node dist/index.js server --url '' --suite all -o results +# 2. Build the manifest from those results: +npm run traceability -- --results results +npm run traceability -- --results results --strict # exit 1 on any untested (advisory) +``` + +Manifest shape: `{ schemaVersion, docs, source, seps }`, where `seps` is keyed by SEP number. Each requirement is `tested` (its check ID was emitted) or `untested` (declared but never emitted — a real gap, or a check that only fires against a deliberately-broken impl, i.e. it needs a negative test). `"tested" means a scenario emitted the check ID, NOT that any SDK passes it` — per-SDK results live in `tier-check`. Matching is exact, so a scenario's emitted check IDs must match the requirement slugs in the yaml (one check ID per MUST/SHOULD, emitted once per case). `source` records what was run against (e.g. `typescript-sdk@`); the `docs` field points back here. + +Contract for consumers (plan.mcp.io): a SEP appears only if it has a traceability yaml or emits `sep-NNNN-*` check IDs. **A SEP absent from the manifest has no conformance artifacts — treat it as not-started** (diff against your own SEP list to find them). `untracked` lists emitted IDs with no yaml row (usually scenario gates). + +The manifest is refreshed by `.github/workflows/traceability.yml` (manual/scheduled), which runs the suite against typescript-sdk and opens a PR with the diff — it is **not** a PR gate. Untested checks are advisory for now; the intended future policy is that an untested check must be backed by a negative test. + ## Examples: prove it passes and fails A new scenario should come with: @@ -101,3 +122,4 @@ Use the existing CLI runner (`npx @modelcontextprotocol/conformance client|serve - `npm test` passes - For non-trivial scenario changes, run against at least one real SDK (typescript-sdk or python-sdk) to see actual output. For changes to shared infrastructure (runner, tier-check), test against go-sdk or csharp-sdk too. - Scenario is registered in the right suite in `src/scenarios/index.ts` +- If you changed a `sep-*.yaml` or scenario check IDs, `src/seps/traceability.json` will drift; the traceability workflow refreshes it via PR (or regenerate locally with `--results` from a suite run) diff --git a/package.json b/package.json index 79535b0a..621436f7 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "lint:fix": "eslint src/ examples/ --fix && prettier --write .", "lint:fix_check": "npm run lint:fix && git diff --exit-code --quiet", "tier-check": "node dist/index.js tier-check", + "traceability": "tsx src/index.ts traceability", "check": "npm run typecheck && npm run lint", "typecheck": "tsgo --noEmit", "prepack": "npm run build", diff --git a/src/index.ts b/src/index.ts index a3431cd3..a3019f0a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,6 +47,7 @@ import { import { createTierCheckCommand } from './tier-check'; import { createNewSepCommand } from './new-sep'; import { createSdkCommand } from './sdk-runner'; +import { createTraceabilityCommand } from './traceability'; import packageJson from '../package.json'; // Note on naming: `command` refers to which CLI command is calling this. @@ -548,6 +549,9 @@ program.addCommand(createNewSepCommand()); // SDK command - run local conformance against an SDK at a specific ref program.addCommand(createSdkCommand()); +// SEP traceability manifest command +program.addCommand(createTraceabilityCommand()); + // List scenarios command program .command('list') diff --git a/src/seps/traceability.json b/src/seps/traceability.json new file mode 100644 index 00000000..5618dd41 --- /dev/null +++ b/src/seps/traceability.json @@ -0,0 +1,409 @@ +{ + "schemaVersion": 1, + "docs": "https://github.com/modelcontextprotocol/conformance/blob/main/AGENTS.md#traceability-manifest", + "source": "typescript-sdk@6f0bf49d", + "seps": { + "2164": { + "yaml": "src/seps/sep-2164.yaml", + "specUrl": "https://modelcontextprotocol.io/specification/draft/server/resources#error-handling", + "requirements": [ + { + "check": "sep-2164-no-empty-contents", + "status": "tested", + "text": "Servers MUST NOT return an empty contents array for a non-existent resource" + }, + { + "check": "sep-2164-error-code", + "status": "tested", + "text": "Servers SHOULD return standard JSON-RPC errors for common failure cases: Resource not found: -32602 (Invalid Params)" + } + ], + "excluded": [ + { + "text": "clients SHOULD also accept -32002 as a resource not found error", + "reason": "Client-side error handling is implementation-defined; not protocol-observable" + } + ], + "unkeyed": [], + "untracked": [ + "sep-2164-data-uri" + ], + "summary": { + "tested": 2, + "untested": 0, + "excluded": 1, + "untracked": 1, + "unkeyed": 0 + } + }, + "2207": { + "yaml": null, + "specUrl": null, + "requirements": [], + "excluded": [], + "unkeyed": [], + "untracked": [ + "sep-2207-client-metadata-grant-types", + "sep-2207-offline-access-not-requested", + "sep-2207-offline-access-requested" + ], + "summary": { + "tested": 0, + "untested": 0, + "excluded": 0, + "untracked": 3, + "unkeyed": 0 + } + }, + "2243": { + "yaml": "src/seps/sep-2243.yaml", + "specUrl": "https://modelcontextprotocol.io/specification/draft/basic/transports#standard-mcp-request-headers", + "requirements": [ + { + "check": "sep-2243-client-includes-standard-headers", + "status": "tested", + "text": "The client MUST include the standard MCP request headers on each POST request. These headers are REQUIRED for compliance." + }, + { + "check": "sep-2243-header-name-case-insensitive", + "status": "tested", + "text": "Clients and servers MUST use case-insensitive comparisons for header names." + }, + { + "check": "sep-2243-server-reject-invalid-headers", + "status": "tested", + "text": "Servers that process the request body MUST reject requests with mismatched or missing standard-header values, returning HTTP 400 Bad Request." + }, + { + "check": "sep-2243-server-reject-error-code", + "status": "tested", + "text": "When rejecting a request due to header validation failure, servers SHOULD include a JSON-RPC error response using error code -32001." + }, + { + "check": "sep-2243-client-supports-custom-headers", + "status": "untested", + "text": "MCP clients MUST support this feature [custom headers via x-mcp-header]." + }, + { + "check": "sep-2243-client-mirrors-designated-params", + "status": "untested", + "text": "When a client invokes a tool whose definition includes such designations, conforming clients MUST mirror the designated parameter values into HTTP headers as described below." + }, + { + "check": "sep-2243-x-mcp-header-not-empty", + "status": "untested", + "text": "The x-mcp-header value MUST NOT be empty.", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#custom-headers" + }, + { + "check": "sep-2243-x-mcp-header-charset", + "status": "untested", + "text": "The x-mcp-header value MUST contain only ASCII characters (excluding space and `:`).", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#custom-headers" + }, + { + "check": "sep-2243-x-mcp-header-unique", + "status": "untested", + "text": "The x-mcp-header value MUST be case-insensitively unique within a single tool definition.", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#custom-headers" + }, + { + "check": "sep-2243-x-mcp-header-primitive-only", + "status": "untested", + "text": "x-mcp-header MUST only be applied to parameters with primitive types (number, string, or boolean).", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#custom-headers" + }, + { + "check": "sep-2243-client-reject-invalid-tool", + "status": "tested", + "text": "Clients MUST reject tool definitions where any x-mcp-header value violates these constraints. Rejection means the client MUST exclude the invalid tool from the set of tools returned by tools/list.", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#custom-headers" + }, + { + "check": "sep-2243-client-encode-values", + "status": "untested", + "text": "Clients MUST encode parameter values before including them in HTTP headers: number values MUST be converted to their decimal string representation; boolean values MUST be converted to the lowercase strings \"true\" or \"false\"." + }, + { + "check": "sep-2243-client-base64-unsafe", + "status": "untested", + "text": "When a value cannot be safely represented as plain ASCII (e.g., contains non-ASCII characters, control characters, or leading/trailing whitespace), clients MUST use Base64 encoding of the UTF-8 representation, wrapped as =?base64?{encoded}?=." + }, + { + "check": "sep-2243-server-decode-base64", + "status": "untested", + "text": "Servers and intermediaries that need to inspect these values MUST decode them accordingly." + }, + { + "check": "sep-2243-client-omit-null", + "status": "tested", + "text": "Parameter value is null or omitted: Client MUST omit the header." + }, + { + "check": "sep-2243-server-not-expect-null", + "status": "untested", + "text": "Parameter value is null or omitted: Server MUST NOT expect the header." + }, + { + "check": "sep-2243-server-reject-missing-required", + "status": "untested", + "text": "Required parameter is omitted: Server MUST reject with JSON-RPC error." + }, + { + "check": "sep-2243-server-reject-invalid-param-chars", + "status": "untested", + "text": "Servers MUST reject requests with a recognized Mcp-Param-{Name} header that contain invalid characters." + }, + { + "check": "sep-2243-server-validate-param-match", + "status": "untested", + "text": "Any server that processes the message body MUST validate that encoded header values, after decoding if Base64-encoded, match the corresponding parameter values in the body." + }, + { + "check": "sep-2243-server-reject-param-mismatch", + "status": "untested", + "text": "Servers MUST reject requests with a 400 Bad Request HTTP status and JSON-RPC error code -32001 if any validation fails." + } + ], + "excluded": [ + { + "text": "Clients SHOULD log a warning when rejecting a tool definition due to invalid x-mcp-header, including the tool name and the reason.", + "reason": "Log output is not wire-observable." + }, + { + "text": "Server developers SHOULD NOT mark sensitive parameters (such as passwords, API keys, tokens, or PII) with x-mcp-header.", + "reason": "Design guidance to humans; not protocol-observable." + }, + { + "text": "Intermediaries MUST return an appropriate HTTP error status for validation failures.", + "reason": "Intermediary requirement; conformance harness tests clients and servers, not intermediaries." + }, + { + "text": "Intermediate servers that do not recognize an Mcp-Param-{Name} header MUST forward it and otherwise ignore it.", + "reason": "Intermediary requirement; conformance harness tests clients and servers, not intermediaries." + } + ], + "unkeyed": [], + "untracked": [ + "sep-2243-invalid-tool-tools-list-gate", + "sep-2243-param-header-tool-call-gate", + "sep-2243-server-accepts-whitespace-header-value", + "sep-2243-server-no-xmcp-tool" + ], + "summary": { + "tested": 6, + "untested": 14, + "excluded": 4, + "untracked": 4, + "unkeyed": 0 + } + }, + "2575": { + "yaml": "src/seps/sep-2575.yaml", + "specUrl": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle", + "requirements": [ + { + "check": "sep-2575-client-populates-meta", + "status": "untested", + "text": "Every client request MUST include the following io.modelcontextprotocol/* fields in _meta: protocolVersion, clientInfo, clientCapabilities.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" + }, + { + "check": "sep-2575-server-rejects-undeclared-capability", + "status": "untested", + "text": "A server MUST NOT rely on capabilities the client has not declared. If processing a request requires a capability the client did not include in io.modelcontextprotocol/clientCapabilities, the server MUST return a MissingRequiredClientCapabilityError (-32003).", + "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" + }, + { + "check": "sep-2575-missing-capability-http-400", + "status": "untested", + "text": "On HTTP, the response status MUST be 400 Bad Request [for MissingRequiredClientCapabilityError].", + "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" + }, + { + "check": "sep-2575-server-tags-subscription-id", + "status": "untested", + "text": "On notifications delivered via a subscriptions/listen stream, the server MUST include io.modelcontextprotocol/subscriptionId in _meta so the client can correlate the notification with the originating subscription request.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" + }, + { + "check": "sep-2575-server-stateless-no-prior-context", + "status": "untested", + "text": "A server MUST NOT treat connection or process identity as a proxy for conversation or session continuity. / Servers MUST NOT rely on prior requests over the same connection to establish context (e.g., capabilities, protocol version, client identity)." + }, + { + "check": "sep-2575-server-stateless-no-connection-reuse-required", + "status": "untested", + "text": "Servers MUST NOT require that a client reuse the same connection to perform related operations." + }, + { + "check": "sep-2575-server-unsupported-version-error", + "status": "untested", + "text": "If the server does not implement the requested version (whether the version is unknown to the server, or is a known version the server has chosen not to support), it MUST respond with an UnsupportedProtocolVersionError listing the versions it does support.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation" + }, + { + "check": "sep-2575-client-retry-supported-version", + "status": "untested", + "text": "The client SHOULD select a mutually supported version from the supported list and retry the request, or surface an error to the user if no compatible version exists.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation" + }, + { + "check": "sep-2575-server-implements-discover", + "status": "untested", + "text": "Servers MUST implement server/discover.", + "url": "https://modelcontextprotocol.io/specification/draft/server/discover" + }, + { + "check": "sep-2575-http-server-no-independent-requests-on-stream", + "status": "untested", + "text": "The server MUST NOT send independent JSON-RPC requests on this stream. Server-to-client interactions are embedded as input requests inside an IncompleteResult.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#receiving-messages-1" + }, + { + "check": "sep-2575-http-server-disconnect-is-cancel", + "status": "untested", + "text": "Closing the SSE response stream MUST be treated by the server as cancellation of that request.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1" + }, + { + "check": "sep-2575-http-server-stops-on-cancel", + "status": "untested", + "text": "The server SHOULD stop work on the cancelled request as soon as practical and MUST NOT send any further messages for it [HTTP].", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#cancellation-1" + }, + { + "check": "sep-2575-http-client-sends-version-header", + "status": "untested", + "text": "Every POST request to the MCP endpoint MUST include an MCP-Protocol-Version header.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" + }, + { + "check": "sep-2575-http-version-header-matches-meta", + "status": "untested", + "text": "The header value MUST match the io.modelcontextprotocol/protocolVersion field carried in the request body _meta.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" + }, + { + "check": "sep-2575-http-server-header-mismatch-400", + "status": "untested", + "text": "If the values do not match, the server MUST reject the request with 400 Bad Request and a HeaderMismatch JSON-RPC error.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" + }, + { + "check": "sep-2575-http-server-unsupported-version-400", + "status": "untested", + "text": "If the server does not implement the requested protocol version, it MUST respond with 400 Bad Request and an UnsupportedProtocolVersionError listing its supported versions.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" + }, + { + "check": "sep-2575-http-server-method-not-found-404", + "status": "untested", + "text": "If the server does not implement the requested RPC method, it MUST respond with 404 Not Found and a JSON-RPC error with code -32601 (Method not found).", + "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" + }, + { + "check": "sep-2575-server-honors-notification-filter", + "status": "untested", + "text": "The server MUST NOT send notification types the client has not explicitly requested.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#opening-a-stream" + }, + { + "check": "sep-2575-server-sends-subscription-ack", + "status": "untested", + "text": "The server MUST send notifications/subscriptions/acknowledged as the first message on the stream.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/utilities/subscriptions#acknowledgment" + }, + { + "check": "sep-2575-client-declares-elicitation-capability", + "status": "untested", + "text": "Clients that support elicitation MUST declare the elicitation capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", + "url": "https://modelcontextprotocol.io/specification/draft/client/elicitation#capabilities" + }, + { + "check": "sep-2575-client-declares-roots-capability", + "status": "untested", + "text": "Clients that support roots MUST declare the roots capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", + "url": "https://modelcontextprotocol.io/specification/draft/client/roots#capabilities" + }, + { + "check": "sep-2575-client-declares-sampling-capability", + "status": "untested", + "text": "Clients that support sampling MUST declare the sampling capability in _meta.io.modelcontextprotocol/clientCapabilities on each request.", + "url": "https://modelcontextprotocol.io/specification/draft/client/sampling#capabilities" + }, + { + "check": "sep-2575-server-declares-prompts-in-discover", + "status": "untested", + "text": "Servers that support prompts MUST declare the prompts capability in their DiscoverResult.", + "url": "https://modelcontextprotocol.io/specification/draft/server/prompts#capabilities" + }, + { + "check": "sep-2575-server-sends-prompts-list-changed-on-subscription", + "status": "untested", + "text": "[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with promptsListChanged: true.", + "url": "https://modelcontextprotocol.io/specification/draft/server/prompts#list-changed-notification" + }, + { + "check": "sep-2575-server-sends-tools-list-changed-on-subscription", + "status": "untested", + "text": "[A server with the listChanged] capability SHOULD send a notification to clients that have opened a subscriptions/listen stream with toolsListChanged: true.", + "url": "https://modelcontextprotocol.io/specification/draft/server/tools#list-changed-notification" + }, + { + "check": "sep-2575-server-no-log-without-loglevel", + "status": "untested", + "text": "The server MUST NOT emit notifications/message for a request that does not include [io.modelcontextprotocol/logLevel in _meta].", + "url": "https://modelcontextprotocol.io/specification/draft/server/utilities/logging#per-request-log-level" + } + ], + "excluded": [ + { + "text": "State that needs to span multiple requests (e.g., long-running tasks, application-level handles) MUST be referenced by an explicit identifier the client passes on each request.", + "reason": "architectural guidance, observable only via subscriptionId/task-id rows already listed" + }, + { + "text": "To distinguish notifications belonging to different concurrent subscriptions, clients MUST correlate notifications using the io.modelcontextprotocol/subscriptionId field carried in _meta.", + "reason": "client-internal demux; not observable on the wire from the harness" + }, + { + "text": "The client SHOULD check the acknowledged filter against what it requested and handle any unsupported types gracefully.", + "reason": "internal comparison; \"gracefully\" has no wire-observable definition" + }, + { + "text": "Because there is no per-request status code to drive fallback, a client that supports both eras SHOULD probe with server/discover first [stdio backward compatibility].", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + }, + { + "text": "To cancel an in-flight request [on stdio], the client MUST send a notifications/cancelled notification referencing the request ID.", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + }, + { + "text": "Servers SHOULD stop work on a cancelled request as soon as practical and MUST NOT send any further messages for it [stdio].", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + }, + { + "text": "If the server process exits unexpectedly, the client SHOULD restart it.", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + }, + { + "text": "If the server returns UnsupportedProtocolVersionError, [the stdio client] SHOULD retry using one of the advertised supportedVersions rather than falling back to initialize.", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + }, + { + "text": "On stdio, if the connection is terminated and then re-established, the client MUST re-send subscriptions/listen to re-establish its subscriptions.", + "reason": "stdio client harness not implemented — see https://github.com/modelcontextprotocol/conformance/issues/258" + } + ], + "unkeyed": [], + "untracked": [], + "summary": { + "tested": 0, + "untested": 26, + "excluded": 9, + "untracked": 0, + "unkeyed": 0 + } + } + } +} diff --git a/src/traceability/index.test.ts b/src/traceability/index.test.ts new file mode 100644 index 00000000..721365e6 --- /dev/null +++ b/src/traceability/index.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from 'vitest'; +import { computeTraceability, DeclaredSep } from './index'; + +const decl = ( + sep: number, + requirements: DeclaredSep['requirements'], + yaml = `src/seps/sep-${sep}.yaml`, + specUrl: string | null = `https://modelcontextprotocol.io/sep-${sep}` +): DeclaredSep => ({ sep, yaml, specUrl, requirements }); + +describe('computeTraceability', () => { + it('marks a declared check tested when its ID was emitted', () => { + const m = computeTraceability({ + declared: [decl(2164, [{ check: 'sep-2164-error-code', text: 'x' }])], + emitted: new Set(['sep-2164-error-code']) + }); + expect(m.seps['2164'].requirements[0]).toEqual({ + check: 'sep-2164-error-code', + text: 'x', + status: 'tested' + }); + expect(m.seps['2164'].summary.tested).toBe(1); + }); + + it('marks a declared check untested when its ID was not emitted', () => { + const m = computeTraceability({ + declared: [decl(2164, [{ check: 'sep-2164-missing' }])], + emitted: new Set() + }); + expect(m.seps['2164'].requirements[0].status).toBe('untested'); + expect(m.seps['2164'].summary.untested).toBe(1); + }); + + it('propagates text, url, and issue onto requirement rows', () => { + const m = computeTraceability({ + declared: [ + decl(2243, [ + { + check: 'sep-2243-x', + text: 'The client MUST do X', + url: 'https://spec/x#y', + issue: 'https://gh/1' + } + ]) + ], + emitted: new Set(['sep-2243-x']) + }); + expect(m.seps['2243'].requirements[0]).toEqual({ + check: 'sep-2243-x', + text: 'The client MUST do X', + url: 'https://spec/x#y', + issue: 'https://gh/1', + status: 'tested' + }); + }); + + it('collects excluded rows with reasons and issue links', () => { + const m = computeTraceability({ + declared: [ + decl(2243, [ + { + text: 'intermediary rule', + excluded: 'not tested', + issue: 'https://x/1' + } + ]) + ], + emitted: new Set() + }); + expect(m.seps['2243'].excluded).toEqual([ + { text: 'intermediary rule', reason: 'not tested', issue: 'https://x/1' } + ]); + expect(m.seps['2243'].requirements).toEqual([]); + }); + + it('lists rows with neither check nor excluded as unkeyed', () => { + const m = computeTraceability({ + declared: [decl(2243, [{ text: 'orphan row' }])], + emitted: new Set() + }); + expect(m.seps['2243'].unkeyed).toEqual([{ text: 'orphan row' }]); + expect(m.seps['2243'].summary.unkeyed).toBe(1); + }); + + it('reports emitted IDs with no yaml row as untracked', () => { + const m = computeTraceability({ + declared: [decl(2164, [{ check: 'sep-2164-error-code' }])], + emitted: new Set(['sep-2164-error-code', 'sep-2164-extra-check']) + }); + expect(m.seps['2164'].untracked).toEqual(['sep-2164-extra-check']); + }); + + it('includes SEPs with emitted IDs but no yaml (tests without traceability)', () => { + const m = computeTraceability({ + declared: [], + emitted: new Set(['sep-2207-offline-access-requested']) + }); + expect(m.seps['2207'].yaml).toBeNull(); + expect(m.seps['2207'].requirements).toEqual([]); + expect(m.seps['2207'].untracked).toEqual([ + 'sep-2207-offline-access-requested' + ]); + }); + + it('sorts SEP keys numerically and stamps schema/meaning/source', () => { + const m = computeTraceability({ + declared: [ + decl(2243, [{ check: 'sep-2243-a' }]), + decl(414, [{ check: 'sep-414-a' }]) + ], + emitted: new Set(), + source: 'typescript-sdk@abc123' + }); + expect(Object.keys(m.seps)).toEqual(['414', '2243']); + expect(m.schemaVersion).toBe(1); + expect(m.docs).toMatch(/^https?:\/\//); + expect(m.source).toBe('typescript-sdk@abc123'); + }); + + it('defaults source to null', () => { + const m = computeTraceability({ declared: [], emitted: new Set() }); + expect(m.source).toBeNull(); + }); +}); diff --git a/src/traceability/index.ts b/src/traceability/index.ts new file mode 100644 index 00000000..055076da --- /dev/null +++ b/src/traceability/index.ts @@ -0,0 +1,316 @@ +import { Command } from 'commander'; +import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs'; +import path from 'path'; +import { parse as parseYaml } from 'yaml'; +import { + TRACEABILITY_SCHEMA_VERSION, + TraceabilityManifest, + ExcludedRequirement, + RequirementTraceability, + SepTraceability, + UnkeyedRequirement +} from './types'; + +const SEPS_DIR = 'src/seps'; +const OUT_FILE = path.join(SEPS_DIR, 'traceability.json'); + +const DOCS = + 'https://github.com/modelcontextprotocol/conformance/blob/main/AGENTS.md#traceability-manifest'; + +// A yaml requirement row (mirrors new-sep's RequirementRow). +interface RawRequirement { + text?: string; + check?: string; + excluded?: string; + issue?: string; + url?: string; +} +interface RawSepYaml { + sep?: number; + spec_url?: string; + requirements?: RawRequirement[]; +} + +export interface DeclaredSep { + sep: number; + yaml: string; + specUrl: string | null; + requirements: RawRequirement[]; +} + +const CHECK_ID_RE = /^sep-\d+-/; + +function sepOf(id: string): number | null { + const m = id.match(/^sep-(\d+)-/); + return m ? Number(m[1]) : null; +} + +/** + * Pure: join declared requirements against the emitted check-ID set into the + * manifest. A requirement is "tested" iff its check ID was emitted by the run. + * No filesystem access — fully testable. + */ +export function computeTraceability(args: { + declared: DeclaredSep[]; + emitted: Set; + source?: string | null; +}): TraceabilityManifest { + const { declared, emitted } = args; + + const emittedBySep = new Map>(); + for (const id of emitted) { + const sep = sepOf(id); + if (sep === null) continue; + let set = emittedBySep.get(sep); + if (!set) emittedBySep.set(sep, (set = new Set())); + set.add(id); + } + + const declaredBySep = new Map(); + for (const d of declared) declaredBySep.set(d.sep, d); + + const allSeps = [ + ...new Set([...declaredBySep.keys(), ...emittedBySep.keys()]) + ].sort((a, b) => a - b); + + const seps: Record = {}; + + for (const sep of allSeps) { + const d = declaredBySep.get(sep); + const emittedIds = emittedBySep.get(sep) ?? new Set(); + + const requirements: RequirementTraceability[] = []; + const excluded: ExcludedRequirement[] = []; + const unkeyed: UnkeyedRequirement[] = []; + const declaredCheckIds = new Set(); + + for (const r of d?.requirements ?? []) { + const check = r.check; + if (check) { + declaredCheckIds.add(check); + requirements.push({ + check, + status: emittedIds.has(check) ? 'tested' : 'untested', + ...(r.text ? { text: r.text } : {}), + ...(r.url ? { url: r.url } : {}), + ...(r.issue ? { issue: r.issue } : {}) + }); + } else if (r.excluded) { + excluded.push({ + text: r.text ?? '', + reason: r.excluded, + ...(r.issue ? { issue: r.issue } : {}) + }); + } else { + unkeyed.push({ text: r.text ?? '' }); + } + } + + // Untracked: emitted IDs not declared in any yaml row. + const untracked = [...emittedIds] + .filter((id) => !declaredCheckIds.has(id)) + .sort(); + + seps[String(sep)] = { + yaml: d?.yaml ?? null, + specUrl: d?.specUrl ?? null, + requirements, + excluded, + unkeyed, + untracked, + summary: { + tested: requirements.filter((r) => r.status === 'tested').length, + untested: requirements.filter((r) => r.status === 'untested').length, + excluded: excluded.length, + untracked: untracked.length, + unkeyed: unkeyed.length + } + }; + } + + return { + schemaVersion: TRACEABILITY_SCHEMA_VERSION, + docs: DOCS, + source: args.source ?? null, + seps + }; +} + +/** Serialize deterministically (sorted SEP keys, trailing newline). */ +export function serializeManifest(manifest: TraceabilityManifest): string { + return JSON.stringify(manifest, null, 2) + '\n'; +} + +// --- filesystem gathering (not unit-tested; thin IO wrappers) ------------- + +/** Recursively collect emitted sep-NNNN-* check IDs from checks.json files. */ +export function collectEmittedIds(resultsDir: string): Set { + const ids = new Set(); + let entries: string[]; + try { + entries = readdirSync(resultsDir, { recursive: true, encoding: 'utf-8' }); + } catch { + return ids; + } + for (const rel of entries) { + if (path.basename(rel) !== 'checks.json') continue; + try { + const arr = JSON.parse(readFileSync(path.join(resultsDir, rel), 'utf8')); + if (!Array.isArray(arr)) continue; + for (const c of arr) { + if (c && typeof c.id === 'string' && CHECK_ID_RE.test(c.id)) + ids.add(c.id); + } + } catch { + // skip unreadable/partial result files + } + } + return ids; +} + +export function gatherDeclared(sepsDir = SEPS_DIR): DeclaredSep[] { + const out: DeclaredSep[] = []; + const files = readdirSync(sepsDir) + .filter((f) => /^sep-\d+\.yaml$/.test(f)) + .sort(); + for (const f of files) { + const full = path.join(sepsDir, f); + const doc = (parseYaml(readFileSync(full, 'utf8')) ?? {}) as RawSepYaml; + const fileSep = Number(f.match(/^sep-(\d+)\.yaml$/)![1]); + if (!Number.isInteger(doc.sep)) { + console.warn(`WARN ${f}: missing/invalid \`sep:\`; skipping`); + continue; + } + if (doc.sep !== fileSep) { + console.warn( + `WARN ${f}: filename SEP ${fileSep} != doc.sep ${doc.sep}; using doc.sep` + ); + } + out.push({ + sep: doc.sep as number, + yaml: full, + specUrl: doc.spec_url ?? null, + requirements: doc.requirements ?? [] + }); + } + return out; +} + +/** Print per-SEP gaps to stderr. */ +function reportGaps(manifest: TraceabilityManifest): void { + for (const [sep, c] of Object.entries(manifest.seps)) { + const untested = c.requirements.filter((r) => r.status === 'untested'); + if (!untested.length && !c.summary.unkeyed && !c.summary.untracked) + continue; + + const bits: string[] = []; + if (untested.length) bits.push(`${untested.length} untested`); + if (c.summary.unkeyed) bits.push(`${c.summary.unkeyed} unkeyed`); + if (c.summary.untracked) bits.push(`${c.summary.untracked} untracked`); + console.error( + `sep-${sep}: ${bits.join(', ')}${c.yaml ? '' : ' (no yaml)'}` + ); + for (const r of untested) console.error(` untested: ${r.check}`); + for (const id of c.untracked) console.error(` untracked: ${id}`); + } +} + +const HELP_EPILOG = ` +"tested" means a scenario emitted the check ID when the conformance suite ran +against the reference SDK — NOT that any SDK passes it. "untested" means the +declared check ID was never emitted (a real gap, or a check that only fires +against a broken impl / a feature the reference SDK has not implemented). + +--results is required: point it at the output of a suite run against the +reference SDK. Produce one with the sdk runner (clones+builds+runs the SDK), +once per side into the same dir: + conformance sdk typescript-sdk@ --mode client --suite all -o + conformance sdk typescript-sdk@ --mode server --suite all --skip-build -o +Check IDs are collected from /**/checks.json. + +--source records what the run was against (e.g. "typescript-sdk@"). +--allow-empty writes even when no check IDs were collected (default: refuse). +--check exits 1 if the on-disk traceability.json differs from a fresh compute. +--strict exits 1 on any untested requirement (advisory for now).`; + +export function createTraceabilityCommand(): Command { + return new Command('traceability') + .description( + 'Generate src/seps/traceability.json: a manifest mapping declared SEP ' + + 'requirements to conformance scenarios that emit their check IDs' + ) + .addHelpText('after', HELP_EPILOG) + .requiredOption( + '--results ', + 'Results dir from a suite run against the reference SDK ' + + '(reads /**/checks.json)' + ) + .option( + '--source ', + 'What the run was against, recorded in the manifest (e.g. typescript-sdk@)' + ) + .option( + '--allow-empty', + 'Write even when zero check IDs were collected (default: refuse)' + ) + .option( + '--check', + 'Do not write; exit 1 if the on-disk traceability.json is stale' + ) + .option('--strict', 'Exit 1 if any declared requirement is untested') + .action((options) => { + if (!existsSync(options.results)) { + console.error(`results dir not found: ${options.results}`); + process.exit(1); + } + const declared = gatherDeclared(); + const emitted = collectEmittedIds(options.results); + + // Guard the footgun: an empty/wrong results dir would mark everything + // untested and silently clobber the manifest. + if (emitted.size === 0 && !options.allowEmpty) { + console.error( + `no sep-NNNN-* check IDs found under ${options.results} — did the ` + + `suite run write checks.json there? Pass --allow-empty to override.` + ); + process.exit(1); + } + + const manifest = computeTraceability({ + declared, + emitted, + source: options.source ?? null + }); + const serialized = serializeManifest(manifest); + const untestedTotal = Object.values(manifest.seps).reduce( + (n, c) => n + c.summary.untested, + 0 + ); + + if (options.check) { + let current = ''; + try { + current = readFileSync(OUT_FILE, 'utf8'); + } catch { + // missing file -> stale + } + if (current !== serialized) { + console.error( + `${OUT_FILE} is out of date. Regenerate with ` + + `\`npm run traceability -- --results ${options.results}\`, ` + + `review with \`git diff ${OUT_FILE}\`, and commit.` + ); + process.exit(1); + } + console.log(`${OUT_FILE} is up to date.`); + } else { + writeFileSync(OUT_FILE, serialized); + console.log( + `wrote ${OUT_FILE}: ${Object.keys(manifest.seps).length} SEP(s)` + ); + reportGaps(manifest); + } + + if (options.strict && untestedTotal > 0) process.exit(1); + }); +} diff --git a/src/traceability/types.ts b/src/traceability/types.ts new file mode 100644 index 00000000..8edc0221 --- /dev/null +++ b/src/traceability/types.ts @@ -0,0 +1,79 @@ +/** + * Shared types for the SEP traceability manifest (src/seps/traceability.json). + * + * IMPORTANT scope note: this manifest records whether a *conformance scenario + * exists* for each declared SEP requirement. It says NOTHING about whether any + * particular SDK passes that scenario — per-SDK pass/fail lives in `tier-check`. + * + * Joining the two is a future goal, NOT possible today: tier-check reports at + * scenario granularity and does not currently expose per-check IDs, while this + * manifest carries check IDs but not scenario names. Wiring plan.mcp.io's two + * feeds together needs one side to add the missing column first. + */ + +export const TRACEABILITY_SCHEMA_VERSION = 1; + +/** Status of a single declared requirement (a yaml `check:` row). */ +export type CheckStatus = + /** A matching check ID was emitted when the conformance suite ran. */ + | 'tested' + /** Declared, but no matching check ID was emitted by any scenario run. */ + | 'untested'; + +export interface RequirementTraceability { + check: string; + status: CheckStatus; + /** The normative sentence from the yaml (for tracker display). */ + text?: string; + /** Per-requirement spec URL from the yaml, if finer than the SEP's specUrl. */ + url?: string; + /** Tracking issue from the yaml, if any. */ + issue?: string; +} + +export interface ExcludedRequirement { + text: string; + reason: string; + issue?: string; +} + +/** A yaml row with neither `check:` nor `excluded:` (an authoring gap). */ +export interface UnkeyedRequirement { + text: string; +} + +export interface SepTraceability { + /** Path to the traceability yaml, or null if scenarios exist but no yaml. */ + yaml: string | null; + /** Spec URL from the yaml's `spec_url`, or null. */ + specUrl: string | null; + requirements: RequirementTraceability[]; + excluded: ExcludedRequirement[]; + unkeyed: UnkeyedRequirement[]; + /** + * Check IDs emitted by the suite run but not declared in any yaml row. + * Usually scenario scaffolding (gates) or extra checks beyond the SEP. + */ + untracked: string[]; + summary: { + tested: number; + untested: number; + excluded: number; + untracked: number; + unkeyed: number; + }; +} + +export interface TraceabilityManifest { + schemaVersion: number; + /** Pointer to where this file's semantics are documented (not prose-in-data). */ + docs: string; + /** + * What the emitted set was collected against, e.g. "typescript-sdk@". + * Provenance for consumers; no wall-clock timestamp so an unchanged run + * produces an empty diff. null when not supplied. + */ + source: string | null; + /** Keyed by SEP number (as a string). */ + seps: Record; +} From 91755d652c3189d1c707bcceab4fedb49ff5f655 Mon Sep 17 00:00:00 2001 From: Yuan Teoh <45984206+Yuan325@users.noreply.github.com> Date: Tue, 19 May 2026 14:40:35 -0700 Subject: [PATCH 06/13] feat: add server conformance tests for SEP-2575 (#271) * feat: add server conformance tests for SEP-2575 * refactor and add new checks * Fix stateless scenario source field and report SKIPPED for inapplicable capability checks - Add the required `source` field (DRAFT) so the scenario typechecks and is selectable via --spec-version. Without it the class did not satisfy the ClientScenario interface and s.source was dereferenced by the spec-version filter at list time. - Teach runCheck about a SKIPPED status and use it for the two client-capability checks when the server does not return -32003, instead of reporting a green PASS for a requirement that was never exercised. * Remove unimplemented placeholder checks from stateless server scenario The subscriptions/listen, statelessness-invariant, list-changed and disconnect-is-cancel checks emitted SUCCESS without probing the server, which inflated coverage in the traceability report. Drop them until they have real assertions; the corresponding rows remain declared in src/seps/sep-2575.yaml so traceability shows them as not-yet-covered. Also fix the checkErrorId helper to push under the sep-2575-http-server-error-jsonrpc-id slug so its failure path actually short-circuits the aggregate SUCCESS check at the end of the scenario. --------- Co-authored-by: Paul Carleton --- .../servers/typescript/everything-server.ts | 152 +++++ src/scenarios/index.ts | 2 + src/scenarios/server/stateless.test.ts | 138 ++++ src/scenarios/server/stateless.ts | 597 ++++++++++++++++++ 4 files changed, 889 insertions(+) create mode 100644 src/scenarios/server/stateless.test.ts create mode 100644 src/scenarios/server/stateless.ts diff --git a/examples/servers/typescript/everything-server.ts b/examples/servers/typescript/everything-server.ts index 91559f8a..acce2429 100644 --- a/examples/servers/typescript/everything-server.ts +++ b/examples/servers/typescript/everything-server.ts @@ -1053,6 +1053,158 @@ app.use( // Handle POST requests - stateful mode app.post('/mcp', async (req, res) => { const sessionId = req.headers['mcp-session-id'] as string | undefined; + const reqVersion = req.headers['mcp-protocol-version'] as string | undefined; + const body = req.body || {}; + const method = body.method; + const id = body.id ?? null; + const params = body.params || {}; + const meta = params._meta; + const metaVersion = meta?.['io.modelcontextprotocol/protocolVersion']; + + if (!sessionId && (reqVersion || meta)) { + // Missing Transport Header Validation Check + if (!reqVersion) { + return res.status(400).json({ + jsonrpc: '2.0', + id, + error: { code: -32001, message: 'Missing MCP-Protocol-Version header' } + }); + } + + // Per-Request Metadata Integrity Checks (Fields verification) + if ( + !meta || + !meta['io.modelcontextprotocol/protocolVersion'] || + !meta['io.modelcontextprotocol/clientInfo'] || + !meta['io.modelcontextprotocol/clientCapabilities'] + ) { + return res.status(200).json({ + jsonrpc: '2.0', + id, + error: { + code: -32602, + message: 'Invalid params: missing _meta or required fields' + } + }); + } + + // Header Mismatch Verification (-32001, HTTP 400) + if (reqVersion !== metaVersion) { + return res.status(400).json({ + jsonrpc: '2.0', + id, + error: { + code: -32001, + message: 'Mismatched MCP-Protocol-Version header' + } + }); + } + + // Protocol Version Negotiation Matrix (-32602, HTTP 400) + if (metaVersion !== 'DRAFT-2026-v1') { + return res.status(400).json({ + jsonrpc: '2.0', + id, + error: { + code: -32602, + message: 'UnsupportedProtocolVersionError', + data: { supported: ['DRAFT-2026-v1'] } + } + }); + } + + if (method === 'server/discover') { + return res.json({ + jsonrpc: '2.0', + id, + result: { + supportedVersions: ['DRAFT-2026-v1'], + capabilities: { + tools: { listChanged: false }, // Explicitly declare capability flags to resolve check assertions + prompts: { listChanged: false } + }, + serverInfo: { name: 'everything-stateless-server', version: '1.0.0' } + } + }); + } + + if (method === 'tools/list') { + return res.json({ + jsonrpc: '2.0', + id, + result: { + tools: [ + { + name: 'test_missing_capability', + description: 'Test tool requiring sampling', + inputSchema: { type: 'object', properties: {} } + } + ] + } + }); + } + + // Mock fallbacks to answer prompts capability matches safely + if (method === 'prompts/list') { + return res.json({ + jsonrpc: '2.0', + id, + result: { prompts: [] } + }); + } + + if (method === 'tools/call') { + const name = params.name; + if (name === 'test_missing_capability') { + const clientCaps = meta['io.modelcontextprotocol/clientCapabilities']; + + // Missing Required Client Capability Check (-32003, HTTP 400) + if (!clientCaps?.sampling) { + return res.status(400).json({ + jsonrpc: '2.0', + id, + error: { + code: -32003, + message: 'MissingRequiredClientCapabilityError', + data: { requiredCapabilities: ['sampling'] } + } + }); + } + return res.json({ + jsonrpc: '2.0', + id, + result: { content: [{ type: 'text', text: 'Success' }] } + }); + } + } + + // Removed Methods per SEP-2575 (Changed status from 200 to 400/404 per Transport Spec) + if ( + [ + 'initialize', + 'ping', + 'logging/setLevel', + 'resources/subscribe', + 'resources/unsubscribe' + ].includes(method) + ) { + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { + code: -32601, + message: 'Method not found: removed stateful RPC' + } + }); + } + + // Generic Fallback Unknown Method Handling (HTTP 404, -32601) + return res.status(404).json({ + jsonrpc: '2.0', + id, + error: { code: -32601, message: 'Method not found' } + }); + } try { let transport: StreamableHTTPServerTransport; diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 4b812d94..e198311d 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -17,6 +17,7 @@ import { RequestMetadataScenario } from './client/request-metadata'; // Import all new server test scenarios import { ServerInitializeScenario } from './server/lifecycle'; +import { ServerStatelessScenario } from './server/stateless'; import { PingScenario, @@ -107,6 +108,7 @@ const pendingClientScenariosList: ClientScenario[] = [ const allClientScenariosList: ClientScenario[] = [ // Lifecycle scenarios new ServerInitializeScenario(), + new ServerStatelessScenario(), // Utilities scenarios new LoggingSetLevelScenario(), diff --git a/src/scenarios/server/stateless.test.ts b/src/scenarios/server/stateless.test.ts new file mode 100644 index 00000000..61535293 --- /dev/null +++ b/src/scenarios/server/stateless.test.ts @@ -0,0 +1,138 @@ +import { ServerStatelessScenario } from './stateless'; +import { describe, test, expect } from 'vitest'; +import { ConformanceCheck } from '../../types'; + +const findCheck = (checks: ConformanceCheck[], id: string) => + checks.find((c) => c.id === id); + +describe('Stateless Server Scenario Negative Tests', () => { + // Inline network mocking helper + function mockFetchTarget( + handler: (reqBody: any, reqHeaders: Record) => any + ) { + global.fetch = async (_url: any, init: any) => { + const body = JSON.parse(init.body); + const headers = init.headers || {}; + const responseConfig = await handler(body, headers); + + return { + status: responseConfig?.status ?? 404, + json: async () => + responseConfig?.body ?? { + jsonrpc: '2.0', + id: body.id, + error: { code: -32601, message: 'Not found' } + } + } as Response; + }; + return 'http://mock-stateless-mcp-server.local'; + } + + test('Fails validation if missing required fields in _meta are allowed to pass', async () => { + // This bad server completely ignores missing params/_meta fields and returns a fake success result + const mockUrl = mockFetchTarget((reqBody) => { + if (reqBody.method === 'server/discover') { + return { + status: 200, + body: { + jsonrpc: '2.0', + id: reqBody.id, + result: { + supportedVersions: ['DRAFT-2026-v1'], + capabilities: {}, + serverInfo: { name: 'bad-meta-server', version: '1.0.0' } + } + } + }; + } + }); + + const scenario = new ServerStatelessScenario(); + const checks = await scenario.run(mockUrl); + + // The test scenario should flag this server as a FAILURE for skipping meta validation + const missingMetaCheck = findCheck( + checks, + 'sep-2575-request-meta-invalid-missing-meta' + ); + const missingVersionCheck = findCheck( + checks, + 'sep-2575-request-meta-invalid-missing-protocol-version' + ); + + expect(missingMetaCheck?.status).toBe('FAILURE'); + expect(missingVersionCheck?.status).toBe('FAILURE'); + }); + + test('Fails validation if removed legacy RPCs do not return HTTP 404 Not Found', async () => { + // This bad server intercepts the removed 'ping' or 'initialize' methods but incorrectly returns HTTP 200 + const mockUrl = mockFetchTarget((reqBody) => { + if ( + [ + 'initialize', + 'ping', + 'logging/setLevel', + 'resources/subscribe', + 'resources/unsubscribe' + ].includes(reqBody.method) + ) { + return { + status: 200, // Spec Violation: Must be HTTP 404 + body: { + jsonrpc: '2.0', + id: reqBody.id, + error: { + code: -32601, + message: 'Method removed but returning HTTP 200' + } + } + }; + } + }); + + const scenario = new ServerStatelessScenario(); + const checks = await scenario.run(mockUrl); + + const pingRouteCheck = findCheck( + checks, + 'sep-2575-http-server-method-not-found-404-ping' + ); + const initializeRouteCheck = findCheck( + checks, + 'sep-2575-http-server-method-not-found-404-initialize' + ); + + expect(pingRouteCheck?.status).toBe('FAILURE'); + expect(initializeRouteCheck?.status).toBe('FAILURE'); + }); + + test('Fails validation when version negotiation returns mismatched supported versions data', async () => { + // This bad server returns an unexpected array of supported versions during negotiation + const mockUrl = mockFetchTarget((reqBody) => { + const meta = reqBody.params?._meta; + if (meta?.['io.modelcontextprotocol/protocolVersion'] === 'v999.0.0') { + return { + status: 400, + body: { + jsonrpc: '2.0', + id: reqBody.id, + error: { + code: -32602, + message: 'Unsupported version', + data: { supported: ['UNEXPECTED-VERSION-STRING-DRIFT'] } // Spec Violation: Mismatches actual versions + } + } + }; + } + }); + + const scenario = new ServerStatelessScenario(); + const checks = await scenario.run(mockUrl); + + const negotiationMatchCheck = findCheck( + checks, + 'sep-2575-server-unsupported-version-error' + ); + expect(negotiationMatchCheck?.status).toBe('FAILURE'); + }); +}); diff --git a/src/scenarios/server/stateless.ts b/src/scenarios/server/stateless.ts new file mode 100644 index 00000000..9c195aa3 --- /dev/null +++ b/src/scenarios/server/stateless.ts @@ -0,0 +1,597 @@ +/** + * Stateless MCP test scenarios for MCP servers (SEP-2575) + */ + +import { + ClientScenario, + ConformanceCheck, + DRAFT_PROTOCOL_VERSION +} from '../../types'; + +const SPEC_REF = [ + { + id: 'SEP-2575', + url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/pull/2575' + } +]; + +export class ServerStatelessScenario implements ClientScenario { + name = 'server-stateless'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = `Test stateless MCP server architecture (SEP-2575). + +**Server Implementation Requirements:** + +**Endpoints**: +- \`server/discover\`: Returns supportedVersions, capabilities, and serverInfo metadata. +- \`tools/call\`: Implement structural test tools like \`test_missing_capability\` requiring explicit capabilities in \`_meta\`. + +**Grouped Specification Requirements**: + +1. **Per-Request _meta Validation (4 Checks)** + - Rejects requests missing \`_meta\` or lacking structural required internal subfields (\`protocolVersion\`, \`clientInfo\`, \`clientCapabilities\`) with a JSON-RPC \`-32602 Invalid params\` error signature. +2. **Discovery & Capabilities (3 Checks)** + - Implements \`server/discover\` mapping exact mandatory protocol elements. + - Dynamically checks prompt capability declaration constraints, validates that active RPC handlers match advertised discovery capacities. +3. **Version Negotiation & Headers (3 Checks)** + - Mismatched or unknown protocol versions must return an \`UnsupportedProtocolVersionError\` (HTTP status code \`400 Bad Request\`) carrying precise version tracking arrays. + - Absent or altered protocol version header metadata must trigger a \`-32001 Header Mismatch\` error with an HTTP 400 boundary state. +4. **Client Capability Constraints (2 Checks)** + - Accessing platform capabilities without explicit declaration drops requests with a \`-32003 MissingRequiredClientCapabilityError\` containing needed capabilities, returning an HTTP status code \`400 Bad Request\`. +5. **Methods & Routing Mechanics (3 Checks)** + - Removed legacy endpoints (\`initialize\`, \`ping\`, \`logging/setLevel\`, etc.) or generic unknown methods must cleanly yield an HTTP status code \`404 Not Found\` alongside a JSON-RPC \`-32601 Method not found\` payload. All error returns must preserve original request ID mappings.`; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + const timestamp = new Date().toISOString(); + + // Executes a validation rule and pushes the structural result metadata. + async function runCheck( + id: string, + name: string, + description: string, + fn: () => + | Promise<{ error?: string; skipped?: boolean; details?: any } | void> + | ({ error?: string; skipped?: boolean; details?: any } | void), + fallbackDetails = {} + ) { + try { + const result = await fn(); + const errorMessage = result?.error; + const status = errorMessage + ? 'FAILURE' + : result?.skipped + ? 'SKIPPED' + : 'SUCCESS'; + + checks.push({ + id, + name, + description, + status, + timestamp, + errorMessage: errorMessage || undefined, + specReferences: SPEC_REF, + details: result?.details || fallbackDetails + }); + } catch (e) { + checks.push({ + id, + name, + description, + status: 'FAILURE', + timestamp, + errorMessage: String(e), + specReferences: SPEC_REF, + details: fallbackDetails + }); + } + } + + // Helper to send raw RPC requests via fetch + const sendRpc = async ( + method: string, + params?: any, + headersOverrides?: Record, + id: string | number | null = 1 + ) => { + const headers = { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'MCP-Protocol-Version': DRAFT_PROTOCOL_VERSION, + ...headersOverrides + }; + + const body = JSON.stringify({ + jsonrpc: '2.0', + id, + method, + ...(params !== undefined ? { params } : {}) + }); + + const res = await fetch(serverUrl, { method: 'POST', headers, body }); + let data: any = null; + try { + data = await res.json(); + } catch { + // Response might not be JSON + } + return { res, data }; + }; + + const validMeta = { + 'io.modelcontextprotocol/protocolVersion': DRAFT_PROTOCOL_VERSION, + 'io.modelcontextprotocol/clientInfo': { + name: 'conformance-client', + version: '1.0.0' + }, + 'io.modelcontextprotocol/clientCapabilities': {} + }; + + // Helper to check JSON-RPC ID matching on error responses + const checkErrorId = (data: any, expectedId: string | number) => { + if (data && data.error) { + if (data.id !== expectedId) { + checks.push({ + id: 'sep-2575-http-server-error-jsonrpc-id', + name: 'HttpServerErrorJsonrpcId', + description: 'All error responses carry the request JSON-RPC id', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Expected error response id ${expectedId}, got ${data.id}`, + specReferences: SPEC_REF + }); + } + } + }; + + // ========================================== + // 1. Per-request _meta Validation (4 Checks) + // ========================================== + const metaValidationTestCases = [ + { + slug: 'missing-meta', + description: + 'Rejects request with missing _meta with -32602 Invalid params', + params: {}, + rpcId: 101 + }, + { + slug: 'missing-protocol-version', + description: + 'Rejects request with _meta missing io.modelcontextprotocol/protocolVersion', + params: { + _meta: { + 'io.modelcontextprotocol/clientInfo': + validMeta['io.modelcontextprotocol/clientInfo'], + 'io.modelcontextprotocol/clientCapabilities': + validMeta['io.modelcontextprotocol/clientCapabilities'] + } + }, + rpcId: 102 + }, + { + slug: 'missing-client-info', + description: + 'Rejects request with _meta missing io.modelcontextprotocol/clientInfo', + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': + validMeta['io.modelcontextprotocol/protocolVersion'], + 'io.modelcontextprotocol/clientCapabilities': + validMeta['io.modelcontextprotocol/clientCapabilities'] + } + }, + rpcId: 103 + }, + { + slug: 'missing-client-capabilities', + description: + 'Rejects request with _meta missing io.modelcontextprotocol/clientCapabilities', + params: { + _meta: { + 'io.modelcontextprotocol/protocolVersion': + validMeta['io.modelcontextprotocol/protocolVersion'], + 'io.modelcontextprotocol/clientInfo': + validMeta['io.modelcontextprotocol/clientInfo'] + } + }, + rpcId: 104 + } + ]; + for (const testCase of metaValidationTestCases) { + await runCheck( + `sep-2575-request-meta-invalid-${testCase.slug}`, + 'RequestMetaInvalid', + testCase.description, + async () => { + const { data } = await sendRpc( + 'server/discover', + testCase.params, + undefined, + testCase.rpcId + ); + checkErrorId(data, testCase.rpcId); + + if (data?.error?.code !== -32602) { + return { + error: `Expected error code -32602, got ${data?.error?.code}`, + details: { fieldIssue: testCase.slug, response: data } + }; + } + return { details: { fieldIssue: testCase.slug, response: data } }; + }, + { fieldIssue: testCase.slug } + ); + } + + // ========================================== + // 2. Discovery & Capabilities (4 Checks) + // ========================================== + let discoverSupportedVersions: string[] = []; + let discoverCapabilities: any = {}; + let discoverResult: any = null; + let discoverRpcError: any = null; + + try { + const { data } = await sendRpc( + 'server/discover', + { _meta: validMeta }, + undefined, + 201 + ); + discoverResult = data?.result; + + if (Array.isArray(discoverResult?.supportedVersions)) { + discoverSupportedVersions = discoverResult.supportedVersions; + } + if ( + discoverResult?.capabilities && + typeof discoverResult.capabilities === 'object' + ) { + discoverCapabilities = discoverResult.capabilities; + } + } catch (e) { + discoverRpcError = e; + } + + await runCheck( + 'sep-2575-server-implements-discover', + 'ServerImplementsDiscover', + 'Servers MUST implement server/discover.', + () => { + if (discoverRpcError) + return { error: `Discovery failed: ${discoverRpcError.message}` }; + if ( + !discoverResult?.supportedVersions || + !discoverResult?.capabilities || + !discoverResult?.serverInfo + ) { + return { + error: 'Missing mandatory fields in discover response setup', + details: { result: discoverResult } + }; + } + return { details: { result: discoverResult } }; + } + ); + + await runCheck( + 'sep-2575-server-declares-prompts-in-discover', + 'ServerDeclaresPromptsInDiscover', + 'Servers that support prompts MUST declare the prompts capability in their DiscoverResult.', + async () => { + if (discoverRpcError) + return { error: `Prerequisite missing: ${discoverRpcError.message}` }; + const { data: promptsData } = await sendRpc( + 'prompts/list', + { _meta: validMeta }, + undefined, + 203 + ); + const methodExists = + promptsData?.result?.prompts || promptsData?.error?.code !== -32601; + + if (methodExists && !discoverCapabilities.prompts) { + return { + error: + 'Server handles prompts but did not declare prompts capability in discover result', + details: { discoverCapabilities } + }; + } + return { details: { discoverCapabilities, response: promptsData } }; + } + ); + + // Dynamic verification helper to check capability consistency against true handlers + await runCheck( + 'sep-2575-discover-capabilities-match-handlers', + 'DiscoverCapabilitiesMatchHandlers', + 'capabilities matches what the server honors on real RPC calls', + async () => { + if (discoverRpcError) + return { + error: `Discovery runtime check failed: ${discoverRpcError.message}` + }; + const { data: toolsData } = await sendRpc( + 'tools/list', + { _meta: validMeta }, + undefined, + 202 + ); + + if (discoverCapabilities.tools) { + const toolsPassed = Array.isArray(toolsData?.result?.tools); + if (!toolsPassed) + return { + error: 'Advertised tools capability but tools/list call failed', + details: { response: toolsData } + }; + } else { + if (toolsData?.error?.code !== -32601) + return { + error: + 'Did not advertise tools capability but tools/list did not yield -32601', + details: { response: toolsData } + }; + } + return { details: { response: toolsData } }; + } + ); + + // ========================================== + // 3. Version Negotiation & Headers (3 Checks) + // ========================================== + const unsupportedMeta = { + ...validMeta, + 'io.modelcontextprotocol/protocolVersion': 'v999.0.0' + }; + const response301 = await sendRpc( + 'server/discover', + { _meta: unsupportedMeta }, + { 'MCP-Protocol-Version': 'v999.0.0' }, + 301 + ).catch(() => null); + const res301: any = response301?.res ?? null; + const data301: any = response301?.data ?? null; + if (data301) checkErrorId(data301, 301); + + await runCheck( + 'sep-2575-server-unsupported-version-error', + 'ServerUnsupportedVersionError', + 'If the server does not implement the requested version (whether the version is unknown to the server, or is a known version the server has chosen not to support), it MUST respond with an UnsupportedProtocolVersionError listing the versions it does support.', + () => { + if (!data301) + return { error: 'Unsupported version invocation failed completely' }; + const errSupportedVersions = data301?.error?.data?.supported; + const hasErrVersions = + Array.isArray(errSupportedVersions) && + errSupportedVersions.length > 0; + + const validMatch = + hasErrVersions && + errSupportedVersions.every((v: string) => + discoverSupportedVersions.includes(v) + ); + if (!validMatch) + return { + error: `Returned supported versions data layout does not correlate to active server metrics: ${JSON.stringify(errSupportedVersions)}` + }; + return { details: { response: data301 } }; + } + ); + + await runCheck( + 'sep-2575-http-server-unsupported-version-400', + 'HttpServerUnsupportedVersion400', + 'If the server does not implement the requested protocol version, it MUST respond with 400 Bad Request and an UnsupportedProtocolVersionError listing its supported versions.', + () => { + if (!res301) + return { error: 'Network transaction context unavailable' }; + if (res301.status !== 400) + return { + error: `Expected HTTP 400 Bad Request, got status code ${res301.status}` + }; + return { details: { response: data301 } }; + } + ); + + const headerMismatchMeta = { + ...validMeta, + 'io.modelcontextprotocol/protocolVersion': 'v999.0.0' + }; + const responseAbsent = await sendRpc( + 'server/discover', + { _meta: headerMismatchMeta }, + { 'MCP-Protocol-Version': 'mismatch.version' }, + 302 + ).catch(() => null); + const resAbsent: any = responseAbsent?.res ?? null; + const dataAbsent: any = responseAbsent?.data ?? null; + + await runCheck( + 'sep-2575-http-server-header-mismatch-400', + 'HttpServerHeaderMismatch400', + 'If the values do not match, the server MUST reject the request with 400 Bad Request and a HeaderMismatch JSON-RPC error.', + () => { + if (!resAbsent) + return { error: 'Header verification endpoint network hit failed' }; + if (resAbsent.status !== 400 || dataAbsent?.error?.code !== -32001) { + return { + error: `Expected HTTP 400 and JSON-RPC error -32001, got status ${resAbsent.status} with code ${dataAbsent?.error?.code}` + }; + } + return { details: { response: dataAbsent } }; + } + ); + + // ========================================== + // 4. Client Capability Constraints (2 Checks) + // ========================================== + const response401 = await sendRpc( + 'tools/call', + { name: 'test_missing_capability', arguments: {}, _meta: validMeta }, + undefined, + 401 + ).catch(() => null); + const res401: any = response401?.res ?? null; + const data401: any = response401?.data ?? null; + if (data401) checkErrorId(data401, 401); + + // Determine if this server actively enforces client capabilities + const serverRequiresCapability = data401?.error?.code === -32003; + + await runCheck( + 'sep-2575-server-rejects-undeclared-capability', + 'ServerRejectsUndeclaredCapability', + 'A server MUST NOT rely on capabilities the client has not declared. If processing a request requires a capability the client did not include in io.modelcontextprotocol/clientCapabilities, the server MUST return a MissingRequiredClientCapabilityError (-32003).', + () => { + if (!res401) + return { + error: + 'Capability checking call sequence timed out or dropped connection' + }; + + if (!serverRequiresCapability) { + // The server didn't return -32003, so this requirement isn't + // exercised for this method. Report SKIPPED rather than a green PASS. + return { + skipped: true, + details: { + note: 'Skipped requirement tracking: Server returned a non-32003 response, indicating it does not require explicit client capability authorization constraints for this method.', + response: data401 + } + }; + } + + // If it DOES return -32003, strictly validate the requirement payload structure + const reqCaps = data401?.error?.data?.requiredCapabilities; + if (!Array.isArray(reqCaps) || !reqCaps.includes('sampling')) { + return { + error: `Server responded with error code -32003 but failed to provide an array containing the expected 'sampling' capability in error.data.requiredCapabilities`, + details: { response: data401 } + }; + } + + return { details: { response: data401 } }; + } + ); + + await runCheck( + 'sep-2575-missing-capability-http-400', + 'MissingCapabilityHttp400', + 'On HTTP, the response status MUST be 400 Bad Request [for MissingRequiredClientCapabilityError].', + () => { + if (!res401) + return { + error: 'Network transport layer layer context failed to instantiate' + }; + + if (!serverRequiresCapability) { + // No -32003 means the HTTP-400 requirement doesn't apply here. + // Report SKIPPED rather than a green PASS. + return { + skipped: true, + details: { + note: 'Skipped status tracking: Server did not return a MissingRequiredClientCapabilityError.', + httpStatus: res401.status + } + }; + } + + // If it did trigger the capability error, it MUST use HTTP 400 + if (res401.status !== 400) { + return { + error: `Expected HTTP status code 400 Bad Request for an undeclared capability error response, got ${res401.status}`, + details: { response: data401 } + }; + } + + return { details: { response: data401 } }; + } + ); + + // ========================================== + // 5. Methods & Routing Mechanics (3 Checks) + // ========================================== + const expectedSlugs = [ + 'initialize', + 'ping', + 'logging/setLevel', + 'resources/subscribe', + 'resources/unsubscribe' + ]; + for (const slug of expectedSlugs) { + const cleanMethodParam = slug.toLowerCase().replace('/', '-'); + const response500 = await sendRpc( + slug, + { _meta: validMeta }, + undefined, + 500 + ).catch(() => null); + const res500: any = response500?.res ?? null; + const data500: any = response500?.data ?? null; + + await runCheck( + `sep-2575-http-server-method-not-found-404-${cleanMethodParam}`, + `HttpServerMethodNotFound404${slug.replace('/', '')}`, + `If the server does not implement the removed RPC method '${slug}', it MUST respond with 404 Not Found and a JSON-RPC error with code -32601 (Method not found).`, + () => { + if (!res500 || !data500) + return { + error: + 'Removed method validation hit dropped connections unexpectedly' + }; + if (res500.status !== 404 || data500?.error?.code !== -32601) { + return { + error: `Expected HTTP 404 and code -32601 for removed methods, got HTTP ${res500.status} and code ${data500?.error?.code}` + }; + } + return { details: { response: data500 } }; + } + ); + } + + // Explicit generic unknown method fallback test + const response601 = await sendRpc( + 'unknown/method', + { _meta: validMeta }, + undefined, + 601 + ).catch(() => null); + const res601: any = response601?.res ?? null; + const data601: any = response601?.data ?? null; + + await runCheck( + 'sep-2575-http-server-method-not-found-404', + 'HttpServerMethodNotFound404', + 'If the server does not implement the requested RPC method, it MUST respond with 404 Not Found and a JSON-RPC error with code -32601 (Method not found).', + () => { + if (!res601 || !data601) + return { + error: 'Unknown fallback test target returned an invalid layout' + }; + if (res601.status !== 404 || data601?.error?.code !== -32601) { + return { + error: `Expected HTTP 404 and JSON-RPC error code -32601, got HTTP ${res601.status} and code ${data601?.error?.code}` + }; + } + return { details: { response: data601 } }; + } + ); + + // Final catchall ensuring JSON-RPC id integrity validation rules ran successfully + if (!checks.some((c) => c.id === 'sep-2575-http-server-error-jsonrpc-id')) { + checks.push({ + id: 'sep-2575-http-server-error-jsonrpc-id', + name: 'HttpServerErrorJsonrpcId', + description: 'All error responses carry the request JSON-RPC id', + status: 'SUCCESS', + timestamp, + specReferences: SPEC_REF + }); + } + + return checks; + } +} From 1ef3cd34bdd41e189ed8f13ee4688ba0b26484f8 Mon Sep 17 00:00:00 2001 From: Den Delimarsky <53200638+localden@users.noreply.github.com> Date: Tue, 19 May 2026 14:59:31 -0700 Subject: [PATCH 07/13] SEP-2352: authorization-server migration scenario (#286) * Add SEP-2352 authorization-server migration scenario SEP-2352 requires that client credentials are bound to the issuing authorization server: when PRM authorization_servers changes to a new issuer, clients MUST re-register and MUST NOT reuse the previous AS's client credentials. - Traceability yaml: 3 checks, 3 excluded (internal state / UI) - New auth/authorization-server-migration scenario (draft suite): two auth servers; PRM flips from AS1 to AS2 after the first authenticated request; AS2 asserts it received a fresh /register and never saw AS1's client_id at /authorize or /token - ConformanceOAuthProvider gains invalidateCredentials and bindIssuer so the everything-client can key credentials by issuer (passing example) - everything-client adds an issuer-aware handler for this scenario that re-reads PRM on each 401 and rebinds before re-authorizing - auth-test-reuse-credentials negative client + vitest case * Drop application_type from DCR metadata (not in SDK OAuthClientMetadata type) --------- Co-authored-by: Paul Carleton --- .../typescript/auth-test-reuse-credentials.ts | 35 +++ .../clients/typescript/everything-client.ts | 71 ++++++ .../helpers/ConformanceOAuthProvider.ts | 28 +++ .../auth/authorization-server-migration.ts | 212 ++++++++++++++++++ src/scenarios/client/auth/index.test.ts | 22 +- src/scenarios/client/auth/index.ts | 4 +- src/seps/sep-2352.yaml | 17 ++ 7 files changed, 387 insertions(+), 2 deletions(-) create mode 100644 examples/clients/typescript/auth-test-reuse-credentials.ts create mode 100644 src/scenarios/client/auth/authorization-server-migration.ts create mode 100644 src/seps/sep-2352.yaml diff --git a/examples/clients/typescript/auth-test-reuse-credentials.ts b/examples/clients/typescript/auth-test-reuse-credentials.ts new file mode 100644 index 00000000..842950e3 --- /dev/null +++ b/examples/clients/typescript/auth-test-reuse-credentials.ts @@ -0,0 +1,35 @@ +/** + * Negative client for SEP-2352: uses a single provider with no issuer keying, + * so when PRM authorization_servers changes it presents the previous AS's + * client_id at the new AS. Expected to trigger + * sep-2352-no-reuse-on-as-change / sep-2352-reregister-on-as-change FAILURE. + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { withOAuthRetry, handle401 } from './helpers/withOAuthRetry'; +import { runAsCli } from './helpers/cliRunner'; + +export async function runClient(serverUrl: string): Promise { + const oauthFetch = withOAuthRetry( + 'auth-test-reuse-credentials', + new URL(serverUrl), + handle401 + )(fetch); + const client = new Client( + { name: 'auth-test-reuse-credentials', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + await client.connect(transport); + await client.listTools(); + await client.callTool({ name: 'test-tool', arguments: {} }); + await transport.close(); +} + +runAsCli( + runClient, + import.meta.url, + 'auth-test-reuse-credentials ' +); diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 650cbf60..64610fe4 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -21,6 +21,11 @@ import { } from '@modelcontextprotocol/sdk/client/auth-extensions.js'; import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import { ClientConformanceContextSchema } from '../../../src/schemas/context.js'; +import { + auth, + extractWWWAuthenticateParams +} from '@modelcontextprotocol/sdk/client/auth.js'; +import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js'; import { withOAuthRetry, withOAuthRetryWithProvider, @@ -270,6 +275,72 @@ registerScenarios( runAuthClient ); +// SEP-2352: a well-behaved client keys credentials by issuer. Before each +// (re-)authorization, fetch PRM and rebind the provider; bindIssuer clears +// stale credentials when authorization_servers has changed so the SDK +// re-registers instead of presenting the previous AS's client_id. +async function runAuthMigrationClient(serverUrl: string): Promise { + const provider = new ConformanceOAuthProvider( + 'http://localhost:3000/callback', + { + client_name: 'auth-migration-client', + redirect_uris: ['http://localhost:3000/callback'] + } + ); + + const issuerAware401: typeof handle401 = async ( + response, + p, + next, + sUrl + ): Promise => { + const { resourceMetadataUrl, scope } = + extractWWWAuthenticateParams(response); + if (resourceMetadataUrl) { + const prm = await (await next(resourceMetadataUrl)).json(); + const issuer = Array.isArray(prm?.authorization_servers) + ? prm.authorization_servers[0] + : undefined; + if (issuer) p.bindIssuer(issuer); + } + let result = await auth(p, { + serverUrl: sUrl, + resourceMetadataUrl, + scope, + fetchFn: next as FetchLike + }); + if (result === 'REDIRECT') { + const code = await p.getAuthCode(); + result = await auth(p, { + serverUrl: sUrl, + resourceMetadataUrl, + scope, + authorizationCode: code, + fetchFn: next as FetchLike + }); + } + }; + + const oauthFetch = withOAuthRetryWithProvider( + provider, + new URL(serverUrl), + issuerAware401 + )(fetch); + const client = new Client( + { name: 'auth-migration-client', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + await client.connect(transport); + await client.listTools(); // phase 1: AS₁ + await client.callTool({ name: 'test-tool', arguments: {} }); // phase 2: AS₂ + await transport.close(); +} + +registerScenario('auth/authorization-server-migration', runAuthMigrationClient); + // ============================================================================ // Elicitation defaults scenario // ============================================================================ diff --git a/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts index aa3aef5a..e6864bfd 100644 --- a/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts +++ b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts @@ -12,6 +12,8 @@ export class ConformanceOAuthProvider implements OAuthClientProvider { private _codeVerifier?: string; private _authCode?: string; private _authCodePromise?: Promise; + /** Issuer the current credentials were obtained from (SEP-2352 keying). */ + private _boundIssuer?: string; constructor( private readonly _redirectUrl: string | URL, @@ -92,4 +94,30 @@ export class ConformanceOAuthProvider implements OAuthClientProvider { } return this._codeVerifier; } + + /** SDK calls this on auth errors; also used by bindIssuer below. */ + invalidateCredentials(scope: 'all' | 'tokens'): void { + this._tokens = undefined; + this._authCode = undefined; + if (scope === 'all') { + this._clientInformation = undefined; + this._codeVerifier = undefined; + } + } + + /** + * SEP-2352: associate stored credentials with the AS issuer that issued them. + * If the issuer changes (PRM migrated to a new authorization server), clear + * everything so the SDK re-registers instead of reusing stale credentials. + * Returns true when a change was detected and credentials were cleared. + */ + bindIssuer(issuer: string): boolean { + if (this._boundIssuer !== undefined && this._boundIssuer !== issuer) { + this.invalidateCredentials('all'); + this._boundIssuer = issuer; + return true; + } + this._boundIssuer = issuer; + return false; + } } diff --git a/src/scenarios/client/auth/authorization-server-migration.ts b/src/scenarios/client/auth/authorization-server-migration.ts new file mode 100644 index 00000000..858bddb5 --- /dev/null +++ b/src/scenarios/client/auth/authorization-server-migration.ts @@ -0,0 +1,212 @@ +/** + * SEP-2352 — Authorization-server binding and migration. + * + * The MCP server's PRM initially lists AS₁. The client registers, authorizes, + * and calls tools/list. The harness then invalidates the token and flips PRM + * to AS₂. On the next 401 the client re-discovers PRM, sees a new issuer, and + * MUST re-register with AS₂ rather than reuse AS₁'s client credentials. + */ +import type { Request, Response, NextFunction } from 'express'; +import type { Scenario, ConformanceCheck } from '../../../types'; +import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { createAuthServer } from './helpers/createAuthServer'; +import { createServer } from './helpers/createServer'; +import { ServerLifecycle } from './helpers/serverLifecycle'; +import { MockTokenVerifier } from './helpers/mockTokenVerifier'; +import { SpecReferences } from './spec-references'; + +export class AuthorizationServerMigrationScenario implements Scenario { + name = 'auth/authorization-server-migration'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that a client, when the PRM authorization_servers changes to a new issuer, re-registers with the new authorization server and does not reuse credentials from the previous one (SEP-2352).'; + private as1 = new ServerLifecycle(); + private as2 = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + + async start(): Promise { + this.checks = []; + const tokenVerifier = new MockTokenVerifier(this.checks, ['mcp:basic']); + + /** AS₁ issues a recognizable client_id so AS₂ can detect cross-AS reuse. */ + const AS1_CLIENT_ID = 'as1-client-id-LEAKED-IF-SEEN-AT-AS2'; + let as2SawRegister = false; + let as2SawAs1ClientId = false; + let as2SawAs1ClientIdAtToken = false; + + // ── AS₁ ──────────────────────────────────────────────────────────────── + const as1App = createAuthServer(this.checks, this.as1.getUrl, { + tokenVerifier, + onRegistrationRequest: () => ({ + clientId: AS1_CLIENT_ID, + clientSecret: 'as1-client-secret' + }) + }); + await this.as1.start(as1App); + + // ── AS₂ ──────────────────────────────────────────────────────────────── + const as2App = createAuthServer(this.checks, this.as2.getUrl, { + tokenVerifier, + onRegistrationRequest: () => { + as2SawRegister = true; + return { clientId: 'as2-client-id', clientSecret: 'as2-client-secret' }; + }, + onAuthorizationRequest: (data) => { + if (data.clientId === AS1_CLIENT_ID) as2SawAs1ClientId = true; + }, + onTokenRequest: (data) => { + const cid = + data.body.client_id ?? + this.basicAuthClientId(data.authorizationHeader); + if (cid === AS1_CLIENT_ID) as2SawAs1ClientIdAtToken = true; + const scopes = data.scope ? data.scope.split(' ') : ['mcp:basic']; + const token = `test-token-as2-${Date.now()}`; + tokenVerifier.registerToken(token, scopes); + return { token, scopes }; + } + }); + await this.as2.start(as2App); + + // ── MCP server with mutable PRM authorization_servers ────────────────── + let migrated = false; + const currentAuthServerUrl = () => + migrated ? this.as2.getUrl() : this.as1.getUrl(); + + const resourceMetadataUrl = () => + `${this.server.getUrl()}/.well-known/oauth-protected-resource/mcp`; + + const middleware = async ( + req: Request, + res: Response, + next: NextFunction + ) => { + let body = req.body; + if (typeof body === 'string') body = JSON.parse(body); + const method = body?.method; + // initialize / notifications never require auth + if (method === 'initialize' || method?.startsWith('notifications/')) + return next(); + + const auth = req.headers.authorization; + if (!auth || !auth.startsWith('Bearer ')) { + return res + .status(401) + .set( + 'WWW-Authenticate', + `Bearer scope="mcp:basic", resource_metadata="${resourceMetadataUrl()}"` + ) + .json({ error: 'invalid_token' }); + } + const token = auth.substring('Bearer '.length); + const info = await tokenVerifier.verifyAccessToken(token); + + // Phase 1: accept any verified token, then flip PRM to AS₂ for the next + // call. Phase 2: reject the (now-stale) AS₁ token so the client + // re-discovers PRM and sees AS₂. + if (!migrated) { + migrated = true; + return next(); + } + // After migration, only AS₂ tokens are valid. + if (!info.token.startsWith('test-token-as2-')) { + return res + .status(401) + .set( + 'WWW-Authenticate', + `Bearer scope="mcp:basic", resource_metadata="${resourceMetadataUrl()}"` + ) + .json({ + error: 'invalid_token', + error_description: 'authorization server has changed' + }); + } + return next(); + }; + + const app = createServer( + this.checks, + this.server.getUrl, + currentAuthServerUrl, + { + prmPath: '/.well-known/oauth-protected-resource/mcp', + requiredScopes: ['mcp:basic'], + includeScopeInWwwAuth: true, + authMiddleware: middleware, + tokenVerifier + } + ); + await this.server.start(app); + + // Record evaluation closures so getChecks can see them after stop(). + this._evaluate = () => { + const ts = new Date().toISOString(); + const reusedAtAS2 = as2SawAs1ClientId || as2SawAs1ClientIdAtToken; + this.checks.push({ + id: 'sep-2352-reregister-on-as-change', + name: 'Client re-registers with the new authorization server', + description: as2SawRegister + ? 'Client performed Dynamic Client Registration with the new authorization server after PRM authorization_servers changed' + : 'Client MUST re-register with the new authorization server when PRM authorization_servers changes (SEP-2352); no registration request was observed at the new AS', + status: as2SawRegister ? 'SUCCESS' : 'FAILURE', + timestamp: ts, + specReferences: [SpecReferences.MCP_DCR] + }); + this.checks.push({ + id: 'sep-2352-no-reuse-on-as-change', + name: 'Client does not reuse the previous AS client credentials', + description: reusedAtAS2 + ? 'Client MUST NOT reuse client credentials from a different authorization server (SEP-2352); the previous AS client_id was observed at the new AS' + : 'Client did not present the previous AS client_id at the new authorization server', + status: reusedAtAS2 ? 'FAILURE' : 'SUCCESS', + timestamp: ts, + specReferences: [SpecReferences.MCP_DCR], + details: { + previousClientId: AS1_CLIENT_ID, + seenAtAuthorize: as2SawAs1ClientId, + seenAtToken: as2SawAs1ClientIdAtToken + } + }); + // The "no cross-AS credential reuse" general MUST NOT is the same wire + // observation as no-reuse-on-as-change in this scenario; emit it as a + // distinct id so the yaml traceability is 1:1. + this.checks.push({ + id: 'sep-2352-no-cross-as-credential-reuse', + name: 'Client does not assume credentials are portable across authorization servers', + description: reusedAtAS2 + ? 'Client MUST NOT assume that credentials valid for one authorization server will be accepted by another (SEP-2352)' + : 'Client treated credentials as bound to the issuing authorization server', + status: reusedAtAS2 ? 'FAILURE' : 'SUCCESS', + timestamp: ts, + specReferences: [SpecReferences.MCP_DCR] + }); + }; + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + private _evaluate: () => void = () => {}; + + private basicAuthClientId(h?: string): string | undefined { + if (!h?.startsWith('Basic ')) return undefined; + try { + const [id] = Buffer.from(h.slice(6), 'base64') + .toString('utf8') + .split(':'); + return id; + } catch { + return undefined; + } + } + + async stop() { + await this.server.stop(); + await this.as1.stop(); + await this.as2.stop(); + } + + getChecks(): ConformanceCheck[] { + this._evaluate(); + return this.checks; + } +} diff --git a/src/scenarios/client/auth/index.test.ts b/src/scenarios/client/auth/index.test.ts index e18a4671..92a72205 100644 --- a/src/scenarios/client/auth/index.test.ts +++ b/src/scenarios/client/auth/index.test.ts @@ -14,6 +14,7 @@ import { runClient as partialScopesClient } from '../../../../examples/clients/t import { runClient as ignore403Client } from '../../../../examples/clients/typescript/auth-test-ignore-403'; import { runClient as noRetryLimitClient } from '../../../../examples/clients/typescript/auth-test-no-retry-limit'; import { runClient as noPkceClient } from '../../../../examples/clients/typescript/auth-test-no-pkce'; +import { runClient as reuseCredsClient } from '../../../../examples/clients/typescript/auth-test-reuse-credentials'; import { getHandler } from '../../../../examples/clients/typescript/everything-client'; import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger'; @@ -29,7 +30,10 @@ const allowClientErrorScenarios = new Set([ // Client is expected to give up (error) after limited retries, but check should pass 'auth/scope-retry-limit', // Client is expected to error when PRM resource doesn't match server URL - 'auth/resource-mismatch' + 'auth/resource-mismatch', + // The post-migration retry path may surface as a client error after + // re-registering; the SEP-2352 checks are evaluated in getChecks() + 'auth/authorization-server-migration' ]); describe('Client Auth Scenarios', () => { @@ -135,6 +139,22 @@ describe('Negative tests', () => { }); }); + test('client reuses credentials across authorization servers (SEP-2352)', async () => { + const runner = new InlineClientRunner(reuseCredsClient); + await runClientAgainstScenario( + runner, + 'auth/authorization-server-migration', + { + allowClientError: true, + expectedFailureSlugs: [ + 'sep-2352-reregister-on-as-change', + 'sep-2352-no-reuse-on-as-change', + 'sep-2352-no-cross-as-credential-reuse' + ] + } + ); + }); + test('client does not use PKCE', async () => { const runner = new InlineClientRunner(noPkceClient); await runClientAgainstScenario(runner, 'auth/metadata-default', { diff --git a/src/scenarios/client/auth/index.ts b/src/scenarios/client/auth/index.ts index 62d60b58..85b3f3f9 100644 --- a/src/scenarios/client/auth/index.ts +++ b/src/scenarios/client/auth/index.ts @@ -28,6 +28,7 @@ import { OfflineAccessScopeScenario, OfflineAccessNotSupportedScenario } from './offline-access'; +import { AuthorizationServerMigrationScenario } from './authorization-server-migration'; // Auth scenarios (required for tier 1) export const authScenariosList: Scenario[] = [ @@ -61,5 +62,6 @@ export const extensionScenariosList: Scenario[] = [ export const draftScenariosList: Scenario[] = [ new ResourceMismatchScenario(), new OfflineAccessScopeScenario(), - new OfflineAccessNotSupportedScenario() + new OfflineAccessNotSupportedScenario(), + new AuthorizationServerMigrationScenario() ]; diff --git a/src/seps/sep-2352.yaml b/src/seps/sep-2352.yaml new file mode 100644 index 00000000..5f025110 --- /dev/null +++ b/src/seps/sep-2352.yaml @@ -0,0 +1,17 @@ +sep: 2352 +spec_url: https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-binding +requirements: + - check: sep-2352-no-cross-as-credential-reuse + text: 'Clients MUST NOT assume that credentials valid for one authorization server will be accepted by another.' + url: https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-location + - check: sep-2352-no-reuse-on-as-change + text: 'When the authorization server changes (detected via updated protected resource metadata), clients MUST NOT reuse client credentials from a different authorization server.' + - check: sep-2352-reregister-on-as-change + text: 'When the authorization server changes (detected via updated protected resource metadata), clients MUST re-register with the new authorization server.' + + - text: 'Clients MUST maintain separate registration state (client credentials, tokens) per authorization server.' + excluded: 'internal storage requirement; not directly observable on the wire' + - text: 'Clients that use pre-registered credentials, or persist client credentials obtained via Dynamic Client Registration, MUST associate those credentials with the specific authorization server that issued them, keyed by the authorization server issuer identifier.' + excluded: 'internal state-keying requirement; not protocol-observable' + - text: 'If the authorization server indicated by protected resource metadata no longer matches the one the credentials were registered with, clients SHOULD surface an error rather than silently attempting to use mismatched credentials.' + excluded: 'UI behavior; the negative half (do not send mismatched credentials) is covered by sep-2352-no-reuse-on-as-change' From 26e09df1543c95a61532e61be9b38c8ea21439a3 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 23:02:24 +0100 Subject: [PATCH 08/13] ci(traceability): enable corepack so the SDK build can use pnpm (#289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refresh run failed with 'pnpm: not found' — the reference SDK's build command (typescript-sdk: pnpm install && pnpm run build:all) needs pnpm on PATH. Add 'corepack enable' to the run job. --- .github/workflows/traceability.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/traceability.yml b/.github/workflows/traceability.yml index 44c85d9f..70f211c8 100644 --- a/.github/workflows/traceability.yml +++ b/.github/workflows/traceability.yml @@ -42,6 +42,10 @@ jobs: node-version: 24 cache: npm + # The SDK's own build (e.g. typescript-sdk uses `pnpm install && pnpm run + # build:all`) needs pnpm on PATH; corepack provides it. + - run: corepack enable + - run: npm ci - run: npm run build From e5afe404ad847fd514d660e6f75c92cd4ddef3ee Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 23:06:39 +0100 Subject: [PATCH 09/13] ci(traceability): tolerate SDK conformance failures in the run step (#290) The `sdk` command exits non-zero when the SDK has conformance failures not in its baseline. The traceability manifest only needs the emitted check IDs (written regardless of pass/fail), so a failing SDK must not fail the refresh. Add `|| true`; the existing 'no results produced' guard catches a genuinely broken run. --- .github/workflows/traceability.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/traceability.yml b/.github/workflows/traceability.yml index 70f211c8..31951057 100644 --- a/.github/workflows/traceability.yml +++ b/.github/workflows/traceability.yml @@ -52,9 +52,12 @@ jobs: - name: Run conformance suites against the reference SDK # `sdk` requires --mode client|server; run both into the same results dir # (the second reuses the cached checkout + build via --skip-build). + # `|| true`: the manifest only needs the emitted check IDs (written + # regardless of pass/fail), so SDK conformance failures must not fail + # this step. The "no results produced" guard below is the real safety net. run: | - node dist/index.js sdk "$SDK_REF" --mode client --suite all -o results - node dist/index.js sdk "$SDK_REF" --mode server --suite all --skip-build -o results + node dist/index.js sdk "$SDK_REF" --mode client --suite all -o results || true + node dist/index.js sdk "$SDK_REF" --mode server --suite all --skip-build -o results || true - name: Fail if no results were produced run: | From e6fffa2b3071aa0148bd6940daf7f663f7434ab3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 23:13:57 +0100 Subject: [PATCH 10/13] chore: refresh SEP traceability manifest (typescript-sdk@main) (#291) Co-authored-by: github-actions[bot] --- src/seps/traceability.json | 83 ++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/src/seps/traceability.json b/src/seps/traceability.json index 5618dd41..e4091c03 100644 --- a/src/seps/traceability.json +++ b/src/seps/traceability.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "docs": "https://github.com/modelcontextprotocol/conformance/blob/main/AGENTS.md#traceability-manifest", - "source": "typescript-sdk@6f0bf49d", + "source": "typescript-sdk@4e153aef0538", "seps": { "2164": { "yaml": "src/seps/sep-2164.yaml", @@ -198,6 +198,51 @@ "unkeyed": 0 } }, + "2352": { + "yaml": "src/seps/sep-2352.yaml", + "specUrl": "https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-binding", + "requirements": [ + { + "check": "sep-2352-no-cross-as-credential-reuse", + "status": "tested", + "text": "Clients MUST NOT assume that credentials valid for one authorization server will be accepted by another.", + "url": "https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-location" + }, + { + "check": "sep-2352-no-reuse-on-as-change", + "status": "tested", + "text": "When the authorization server changes (detected via updated protected resource metadata), clients MUST NOT reuse client credentials from a different authorization server." + }, + { + "check": "sep-2352-reregister-on-as-change", + "status": "tested", + "text": "When the authorization server changes (detected via updated protected resource metadata), clients MUST re-register with the new authorization server." + } + ], + "excluded": [ + { + "text": "Clients MUST maintain separate registration state (client credentials, tokens) per authorization server.", + "reason": "internal storage requirement; not directly observable on the wire" + }, + { + "text": "Clients that use pre-registered credentials, or persist client credentials obtained via Dynamic Client Registration, MUST associate those credentials with the specific authorization server that issued them, keyed by the authorization server issuer identifier.", + "reason": "internal state-keying requirement; not protocol-observable" + }, + { + "text": "If the authorization server indicated by protected resource metadata no longer matches the one the credentials were registered with, clients SHOULD surface an error rather than silently attempting to use mismatched credentials.", + "reason": "UI behavior; the negative half (do not send mismatched credentials) is covered by sep-2352-no-reuse-on-as-change" + } + ], + "unkeyed": [], + "untracked": [], + "summary": { + "tested": 3, + "untested": 0, + "excluded": 3, + "untracked": 0, + "unkeyed": 0 + } + }, "2575": { "yaml": "src/seps/sep-2575.yaml", "specUrl": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle", @@ -210,13 +255,13 @@ }, { "check": "sep-2575-server-rejects-undeclared-capability", - "status": "untested", + "status": "tested", "text": "A server MUST NOT rely on capabilities the client has not declared. If processing a request requires a capability the client did not include in io.modelcontextprotocol/clientCapabilities, the server MUST return a MissingRequiredClientCapabilityError (-32003).", "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" }, { "check": "sep-2575-missing-capability-http-400", - "status": "untested", + "status": "tested", "text": "On HTTP, the response status MUST be 400 Bad Request [for MissingRequiredClientCapabilityError].", "url": "https://modelcontextprotocol.io/specification/draft/basic/index#meta" }, @@ -238,7 +283,7 @@ }, { "check": "sep-2575-server-unsupported-version-error", - "status": "untested", + "status": "tested", "text": "If the server does not implement the requested version (whether the version is unknown to the server, or is a known version the server has chosen not to support), it MUST respond with an UnsupportedProtocolVersionError listing the versions it does support.", "url": "https://modelcontextprotocol.io/specification/draft/basic/lifecycle#protocol-version-negotiation" }, @@ -250,7 +295,7 @@ }, { "check": "sep-2575-server-implements-discover", - "status": "untested", + "status": "tested", "text": "Servers MUST implement server/discover.", "url": "https://modelcontextprotocol.io/specification/draft/server/discover" }, @@ -286,19 +331,19 @@ }, { "check": "sep-2575-http-server-header-mismatch-400", - "status": "untested", + "status": "tested", "text": "If the values do not match, the server MUST reject the request with 400 Bad Request and a HeaderMismatch JSON-RPC error.", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" }, { "check": "sep-2575-http-server-unsupported-version-400", - "status": "untested", + "status": "tested", "text": "If the server does not implement the requested protocol version, it MUST respond with 400 Bad Request and an UnsupportedProtocolVersionError listing its supported versions.", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" }, { "check": "sep-2575-http-server-method-not-found-404", - "status": "untested", + "status": "tested", "text": "If the server does not implement the requested RPC method, it MUST respond with 404 Not Found and a JSON-RPC error with code -32601 (Method not found).", "url": "https://modelcontextprotocol.io/specification/draft/basic/transports#protocol-version-header" }, @@ -334,7 +379,7 @@ }, { "check": "sep-2575-server-declares-prompts-in-discover", - "status": "untested", + "status": "tested", "text": "Servers that support prompts MUST declare the prompts capability in their DiscoverResult.", "url": "https://modelcontextprotocol.io/specification/draft/server/prompts#capabilities" }, @@ -396,12 +441,24 @@ } ], "unkeyed": [], - "untracked": [], + "untracked": [ + "sep-2575-discover-capabilities-match-handlers", + "sep-2575-http-server-error-jsonrpc-id", + "sep-2575-http-server-method-not-found-404-initialize", + "sep-2575-http-server-method-not-found-404-logging-setlevel", + "sep-2575-http-server-method-not-found-404-ping", + "sep-2575-http-server-method-not-found-404-resources-subscribe", + "sep-2575-http-server-method-not-found-404-resources-unsubscribe", + "sep-2575-request-meta-invalid-missing-client-capabilities", + "sep-2575-request-meta-invalid-missing-client-info", + "sep-2575-request-meta-invalid-missing-meta", + "sep-2575-request-meta-invalid-missing-protocol-version" + ], "summary": { - "tested": 0, - "untested": 26, + "tested": 8, + "untested": 18, "excluded": 9, - "untracked": 0, + "untracked": 11, "unkeyed": 0 } } From 708a0dd4bd9e4b1eb4d9eebf0930fec4363bea33 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Tue, 19 May 2026 23:23:29 +0100 Subject: [PATCH 11/13] =?UTF-8?q?Make=20scopesSupported=20disjoint=20from?= =?UTF-8?q?=20challenge=20scopes=20so=20the=20SEP-2350=20union=20check=20c?= =?UTF-8?q?an't=20be=20satisfied=20by=20scopes=5Fsupported=20=E2=88=AA=20c?= =?UTF-8?q?hallenge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A client that unions scopes_supported with the challenge — instead of its prior grant with the challenge — would have requested mcp:basic mcp:write and falsely passed sep-2350-scope-union-on-reauth, since scopesSupported happened to coincide with the previously-granted scope. Setting it to a disjoint value makes the check actually require the prior token's scope. Spec backing: clients MUST NOT assume any set relationship between the challenged scope set and scopes_supported, so a disjoint advertisement is realistic. --- src/scenarios/client/auth/scope-handling.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index 9b6c9f68..c24d51d6 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -443,7 +443,12 @@ export class ScopeStepUpAuthScenario implements Scenario { { prmPath: '/.well-known/oauth-protected-resource/mcp', requiredScopes: escalatedScopes, - scopesSupported: [initialScope], + // Deliberately disjoint from initialScope/stepUpScope so the SEP-2350 + // union check can't be satisfied by a client that unions + // scopes_supported ∪ challenge instead of prior-grant ∪ challenge. + // The spec allows this: clients MUST NOT assume any set relationship + // between challenged scopes and scopes_supported. + scopesSupported: ['mcp:profile'], includeScopeInWwwAuth: true, authMiddleware: stepUpMiddleware, tokenVerifier From 12252466e661963e956e0091b350214612f3e11b Mon Sep 17 00:00:00 2001 From: Den Delimarsky <53200638+localden@users.noreply.github.com> Date: Tue, 19 May 2026 15:28:50 -0700 Subject: [PATCH 12/13] SEP-837: application_type check in DCR registration (#284) * Add SEP-837 application_type check to DCR registration SEP-837 requires MCP clients to specify an appropriate application_type during Dynamic Client Registration so OIDC authorization servers can apply the correct redirect-URI constraints. - Traceability yaml: 1 check (presence + valid value), 4 excluded (class-specific SHOULDs unobservable; UI/robustness) - Check added in the shared createAuthServer DCR handler so it fires in every auth scenario that performs DCR (no new scenario) - withOAuthRetry now sets application_type: native (passing example; the conformance example clients are CLI tools) - New auth-test-no-application-type negative client + vitest case * Widen ConformanceOAuthProvider metadata type for application_type Fixes CI typecheck (TS2353 at withOAuthRetry.ts:76): the SDK's OAuthClientMetadataSchema doesn't include application_type yet. registerClient() spreads clientMetadata verbatim into the /register POST body, so a local type intersection is sufficient to get the field on the wire without an SDK release. * Set application_type in runAuthMigrationClient The SEP-2352 authorization-server-migration handler constructs its own ConformanceOAuthProvider (not via withOAuthRetry), so it was missing application_type after rebasing onto main. The scenario does DCR twice (old AS, new AS) and the SEP-837 check fires on each. --------- Co-authored-by: Paul Carleton --- .../auth-test-no-application-type.ts | 44 +++++++++++++++++++ .../clients/typescript/everything-client.ts | 3 +- .../helpers/ConformanceOAuthProvider.ts | 14 +++++- .../typescript/helpers/withOAuthRetry.ts | 5 ++- .../client/auth/helpers/createAuthServer.ts | 19 ++++++++ src/scenarios/client/auth/index.test.ts | 8 ++++ src/seps/sep-837.yaml | 14 ++++++ 7 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 examples/clients/typescript/auth-test-no-application-type.ts create mode 100644 src/seps/sep-837.yaml diff --git a/examples/clients/typescript/auth-test-no-application-type.ts b/examples/clients/typescript/auth-test-no-application-type.ts new file mode 100644 index 00000000..e25ccd85 --- /dev/null +++ b/examples/clients/typescript/auth-test-no-application-type.ts @@ -0,0 +1,44 @@ +/** + * Negative client for SEP-837: registers via DCR WITHOUT application_type. + * Expected to trigger sep-837-application-type-present = FAILURE. + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { + withOAuthRetryWithProvider, + handle401 +} from './helpers/withOAuthRetry'; +import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider'; +import { runAsCli } from './helpers/cliRunner'; + +export async function runClient(serverUrl: string): Promise { + const provider = new ConformanceOAuthProvider( + 'http://localhost:3000/callback', + { + client_name: 'auth-test-no-application-type', + redirect_uris: ['http://localhost:3000/callback'] + // application_type intentionally omitted + } + ); + const oauthFetch = withOAuthRetryWithProvider( + provider, + new URL(serverUrl), + handle401 + )(fetch); + const client = new Client( + { name: 'auth-test-no-application-type', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + await client.connect(transport); + await client.listTools(); + await transport.close(); +} + +runAsCli( + runClient, + import.meta.url, + 'auth-test-no-application-type ' +); diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 64610fe4..71c4582d 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -284,7 +284,8 @@ async function runAuthMigrationClient(serverUrl: string): Promise { 'http://localhost:3000/callback', { client_name: 'auth-migration-client', - redirect_uris: ['http://localhost:3000/callback'] + redirect_uris: ['http://localhost:3000/callback'], + application_type: 'native' } ); diff --git a/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts index e6864bfd..5dffd3d7 100644 --- a/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts +++ b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts @@ -6,6 +6,16 @@ import { OAuthTokens } from '@modelcontextprotocol/sdk/shared/auth.js'; +/** + * SEP-837 adds `application_type` to DCR; the SDK's OAuthClientMetadataSchema + * doesn't include it yet. The SDK spreads clientMetadata verbatim into the + * /register POST body, so widening the type here is sufficient to get the + * field on the wire. Drop this once the SDK schema is updated. + */ +type ConformanceClientMetadata = OAuthClientMetadata & { + application_type?: 'native' | 'web'; +}; + export class ConformanceOAuthProvider implements OAuthClientProvider { private _clientInformation?: OAuthClientInformationFull; private _tokens?: OAuthTokens; @@ -17,7 +27,7 @@ export class ConformanceOAuthProvider implements OAuthClientProvider { constructor( private readonly _redirectUrl: string | URL, - private readonly _clientMetadata: OAuthClientMetadata, + private readonly _clientMetadata: ConformanceClientMetadata, private readonly _clientMetadataUrl?: string | URL ) {} @@ -25,7 +35,7 @@ export class ConformanceOAuthProvider implements OAuthClientProvider { return this._redirectUrl; } - get clientMetadata(): OAuthClientMetadata { + get clientMetadata(): ConformanceClientMetadata { return this._clientMetadata; } diff --git a/examples/clients/typescript/helpers/withOAuthRetry.ts b/examples/clients/typescript/helpers/withOAuthRetry.ts index 429ca539..3d5c8986 100644 --- a/examples/clients/typescript/helpers/withOAuthRetry.ts +++ b/examples/clients/typescript/helpers/withOAuthRetry.ts @@ -70,7 +70,10 @@ export const withOAuthRetry = ( 'http://localhost:3000/callback', { client_name: clientName, - redirect_uris: ['http://localhost:3000/callback'] + redirect_uris: ['http://localhost:3000/callback'], + // SEP-837: the conformance example clients are CLI tools, i.e. native + // applications under the OIDC client-type taxonomy. + application_type: 'native' }, clientMetadataUrl ); diff --git a/src/scenarios/client/auth/helpers/createAuthServer.ts b/src/scenarios/client/auth/helpers/createAuthServer.ts index c8c4ecd2..00aa3dbc 100644 --- a/src/scenarios/client/auth/helpers/createAuthServer.ts +++ b/src/scenarios/client/auth/helpers/createAuthServer.ts @@ -386,6 +386,25 @@ export function createAuthServer( } }); + // SEP-837: clients MUST specify an appropriate application_type during DCR. + // The harness can't know the client's real class (native vs web), so this + // checks presence + that the value is one of the two OIDC-defined values. + const appType = req.body.application_type; + const validAppType = appType === 'native' || appType === 'web'; + checks.push({ + id: 'sep-837-application-type-present', + name: 'DCR application_type specified', + description: validAppType + ? `Client specified application_type "${appType}" during Dynamic Client Registration` + : appType === undefined + ? 'Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); field was omitted' + : `Client MUST specify an appropriate application_type during Dynamic Client Registration (SEP-837); got "${appType}", expected "native" or "web"`, + status: validAppType ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_DCR], + details: { application_type: appType ?? '(omitted)' } + }); + res.status(201).json({ client_id: clientId, ...(clientSecret && { client_secret: clientSecret }), diff --git a/src/scenarios/client/auth/index.test.ts b/src/scenarios/client/auth/index.test.ts index 92a72205..c5d79811 100644 --- a/src/scenarios/client/auth/index.test.ts +++ b/src/scenarios/client/auth/index.test.ts @@ -15,6 +15,7 @@ import { runClient as ignore403Client } from '../../../../examples/clients/types import { runClient as noRetryLimitClient } from '../../../../examples/clients/typescript/auth-test-no-retry-limit'; import { runClient as noPkceClient } from '../../../../examples/clients/typescript/auth-test-no-pkce'; import { runClient as reuseCredsClient } from '../../../../examples/clients/typescript/auth-test-reuse-credentials'; +import { runClient as noAppTypeClient } from '../../../../examples/clients/typescript/auth-test-no-application-type'; import { getHandler } from '../../../../examples/clients/typescript/everything-client'; import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger'; @@ -155,6 +156,13 @@ describe('Negative tests', () => { ); }); + test('client omits application_type during DCR (SEP-837)', async () => { + const runner = new InlineClientRunner(noAppTypeClient); + await runClientAgainstScenario(runner, 'auth/metadata-default', { + expectedFailureSlugs: ['sep-837-application-type-present'] + }); + }); + test('client does not use PKCE', async () => { const runner = new InlineClientRunner(noPkceClient); await runClientAgainstScenario(runner, 'auth/metadata-default', { diff --git a/src/seps/sep-837.yaml b/src/seps/sep-837.yaml new file mode 100644 index 00000000..d97d00a3 --- /dev/null +++ b/src/seps/sep-837.yaml @@ -0,0 +1,14 @@ +sep: 837 +spec_url: https://modelcontextprotocol.io/specification/draft/basic/authorization#application-type-and-redirect-uri-constraints +requirements: + - check: sep-837-application-type-present + text: 'MCP clients MUST specify an appropriate application_type during Dynamic Client Registration.' + + - text: 'Native applications (desktop applications, mobile apps, CLI tools, and locally-hosted web applications accessed via localhost) SHOULD use application_type: "native".' + excluded: 'harness cannot determine the client-under-test application class (native vs web) out-of-band; only presence and value validity are wire-observable' + - text: 'Web applications (remote browser-based applications served from a non-local host) SHOULD use application_type: "web".' + excluded: 'harness cannot determine the client-under-test application class (native vs web) out-of-band; only presence and value validity are wire-observable' + - text: 'MCP clients MUST be prepared to handle registration failures due to redirect URI constraints when authorization servers implement OIDC.' + excluded: 'robustness requirement with no defined wire-level success criterion' + - text: 'When a registration request is rejected, clients SHOULD surface a meaningful error to the user or developer.' + excluded: 'UI/DX behavior, not protocol-observable' From 78d7219198ddbfc63632d5e17f5b9dc8bc7a7eb8 Mon Sep 17 00:00:00 2001 From: Max Gerber <89937743+max-stytch@users.noreply.github.com> Date: Tue, 19 May 2026 15:42:35 -0700 Subject: [PATCH 13/13] feat: add conformance tests for iss parameter (SEP-2468) (#220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add conformance tests for iss parameter (SEP-2468) Adds 5 draft conformance scenarios testing RFC 9207 issuer parameter validation in OAuth authorization responses: - auth/iss-supported: server advertises support and sends correct iss - auth/iss-not-advertised: server omits iss parameter entirely - auth/iss-supported-missing: client must reject missing iss when required - auth/iss-wrong-issuer: client must reject mismatched iss value - auth/iss-unexpected: client must reject iss when not advertised Also adds auth-test-iss-validation.ts, a reference client that correctly validates iss per RFC 9207, and negative tests confirming the standard client fails all three rejection scenarios. TODO: Update RFC_9207_ISS_PARAMETER spec reference once SEP-2468 (modelcontextprotocol/modelcontextprotocol#2468) is merged. * update scenarios * fix: createAuthServer iss option type/guard and NotAdvertised scenario duplication The doc comments said 'Default: not included' but the destructure defaulted to true/'correct', and the `!== undefined` guard at L155 was unreachable — so there was no way to omit the metadata field, and IssParameterNotAdvertised silently advertised support (a duplicate of IssParameterSupported). Kept the on-by-default behavior (mock AS models a well-behaved server) but made issParameterSupported `boolean | null` so callers pass null to omit, matching the codeChallengeMethodsSupported pattern. Doc comments now match. Scenarios that need omission pass null/'omit' explicitly. * fix: rejection scenarios silently pass when client never reaches auth endpoint correctlyRejected = !tokenRequestMade reports SUCCESS if the client errors out before hitting /authorize. Gate on authReached so a setup failure shows as FAILURE with authReached:false in details. * fix: iss-unexpected scenario contradicts SEP-2468 spec table row 3 The spec table says: supported=false/absent + iss present -> *Compare* to the recorded issuer (not reject). The scenario sent a *correct* iss and FAILed compliant clients for proceeding after a successful comparison. Now sends a mismatched iss so the comparison fails and rejection is the spec-required outcome. Reference client updated to compare-when-present instead of throw-on-presence. * refactor: replace harness-config checks with client-proceeded checks iss-advertised-in-metadata / iss-sent-in-redirect (and the not-* variants) fired in onAuthorizationRequest before the redirect happened, asserting only that the harness was configured correctly — a client that ignores iss passes identically. Replaced with one check per scenario keyed on tokenRequestMade, which observes that the client actually proceeded through the iss path. * refactor: rename check IDs to sep-2468-* and align with spec table rows One ID per spec table row; auth/iss-supported and auth/iss-wrong-issuer both emit sep-2468-client-compare-iss-supported (same comparison, opposite input) per the same-slug-for-SUCCESS-and-FAIL convention. * feat: add sep-2468.yaml requirement traceability 8 check rows (4 client table-row checks, 1 metadata-issuer, 2 AS-side, 1 no-normalization), 1 excluded (error-display is UI-facing). The record-issuer MUST is merged into the compare-iss-supported row text since it has no independent wire observation. * fix: migrate iss scenarios specVersions->source (post-#265) Replaces `specVersions: ['draft']` with `source: { introducedIn: DRAFT_PROTOCOL_VERSION }` in the 5 iss-parameter scenarios. This commit typechecks once the stack is rebased onto main >= #265 (the ScenarioSource migration). Adding it now so the rebase is mechanical. * fix: include application_type in iss-validation example DCR (post-#284) The SEP-837 application_type check now runs in every auth scenario; the hand-rolled DCR in auth-test-iss-validation.ts was omitting the field. --------- Co-authored-by: Paul Carleton --- .../typescript/auth-test-iss-validation.ts | 230 ++++++++++ .../clients/typescript/everything-client.ts | 16 +- .../client/auth/helpers/createAuthServer.ts | 16 + src/scenarios/client/auth/index.test.ts | 31 +- src/scenarios/client/auth/index.ts | 14 +- src/scenarios/client/auth/issuer-parameter.ts | 408 ++++++++++++++++++ src/scenarios/client/auth/spec-references.ts | 5 + src/seps/sep-2468.yaml | 23 + 8 files changed, 740 insertions(+), 3 deletions(-) create mode 100644 examples/clients/typescript/auth-test-iss-validation.ts create mode 100644 src/scenarios/client/auth/issuer-parameter.ts create mode 100644 src/seps/sep-2468.yaml diff --git a/examples/clients/typescript/auth-test-iss-validation.ts b/examples/clients/typescript/auth-test-iss-validation.ts new file mode 100644 index 00000000..3fdd3f05 --- /dev/null +++ b/examples/clients/typescript/auth-test-iss-validation.ts @@ -0,0 +1,230 @@ +#!/usr/bin/env node + +/** + * Well-behaved client that validates the iss parameter in authorization responses. + * + * Per RFC 9207: + * - If the AS advertises authorization_response_iss_parameter_supported: true, + * the client MUST require iss in the redirect and MUST validate it against + * the AS metadata issuer. + * - If the AS does NOT advertise support, the client MUST reject any redirect + * that unexpectedly contains an iss parameter. + */ + +import { createHash, randomBytes } from 'crypto'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { extractWWWAuthenticateParams } from '@modelcontextprotocol/sdk/client/auth.js'; +import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js'; +import type { Middleware } from '@modelcontextprotocol/sdk/client/middleware.js'; +import { runAsCli } from './helpers/cliRunner'; +import { logger } from './helpers/logger'; + +interface OAuthTokens { + access_token: string; + token_type: string; + expires_in?: number; + refresh_token?: string; + scope?: string; +} + +function generateCodeVerifier(): string { + return randomBytes(32) + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); +} + +function computeS256Challenge(codeVerifier: string): string { + const hash = createHash('sha256').update(codeVerifier).digest(); + return hash + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/, ''); +} + +/** + * OAuth flow that correctly validates the iss parameter per RFC 9207. + */ +async function oauthFlowWithIssValidation( + _serverUrl: string | URL, + resourceMetadataUrl: string | URL, + fetchFn: FetchLike +): Promise { + // 1. Fetch Protected Resource Metadata + const prmResponse = await fetchFn(resourceMetadataUrl); + if (!prmResponse.ok) { + throw new Error(`Failed to fetch PRM: ${prmResponse.status}`); + } + const prm = await prmResponse.json(); + const authServerUrl = prm.authorization_servers?.[0]; + if (!authServerUrl) { + throw new Error('No authorization server in PRM'); + } + + // 2. Fetch Authorization Server Metadata + const asMetadataUrl = new URL( + '/.well-known/oauth-authorization-server', + authServerUrl + ); + const asResponse = await fetchFn(asMetadataUrl.toString()); + if (!asResponse.ok) { + throw new Error(`Failed to fetch AS metadata: ${asResponse.status}`); + } + const asMetadata = await asResponse.json(); + + const expectedIssuer: string = asMetadata.issuer; + const issParameterSupported: boolean = + asMetadata.authorization_response_iss_parameter_supported === true; + + // 3. Register client (DCR) + const dcrResponse = await fetchFn(asMetadata.registration_endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + client_name: 'test-auth-client-iss-validation', + redirect_uris: ['http://localhost:3000/callback'], + application_type: 'native' + }) + }); + if (!dcrResponse.ok) { + throw new Error(`DCR failed: ${dcrResponse.status}`); + } + const clientInfo = await dcrResponse.json(); + + // 4. Build authorization URL with PKCE + const codeVerifier = generateCodeVerifier(); + const codeChallenge = computeS256Challenge(codeVerifier); + + const authUrl = new URL(asMetadata.authorization_endpoint); + authUrl.searchParams.set('response_type', 'code'); + authUrl.searchParams.set('client_id', clientInfo.client_id); + authUrl.searchParams.set('redirect_uri', 'http://localhost:3000/callback'); + authUrl.searchParams.set('state', 'test-state'); + authUrl.searchParams.set('code_challenge', codeChallenge); + authUrl.searchParams.set('code_challenge_method', 'S256'); + + // 5. Fetch authorization endpoint (simulates redirect) + const authResponse = await fetchFn(authUrl.toString(), { + redirect: 'manual' + }); + const location = authResponse.headers.get('location'); + if (!location) { + throw new Error('No redirect from authorization endpoint'); + } + const redirectUrl = new URL(location); + const authCode = redirectUrl.searchParams.get('code'); + if (!authCode) { + throw new Error('No auth code in redirect'); + } + + // 6. Validate iss parameter per RFC 9207 + const issInRedirect = redirectUrl.searchParams.get('iss'); + + if (issParameterSupported) { + // Server advertised support: iss MUST be present and MUST match metadata issuer + if (!issInRedirect) { + throw new Error( + 'Server advertised authorization_response_iss_parameter_supported but iss is absent from redirect' + ); + } + if (issInRedirect !== expectedIssuer) { + throw new Error( + `iss mismatch: expected '${expectedIssuer}', got '${issInRedirect}'` + ); + } + } else { + // Server did NOT advertise support: if iss is present, compare anyway + // (SEP-2468 spec table row 3 — local-policy provision per RFC 9207 §2.4) + if (issInRedirect && issInRedirect !== expectedIssuer) { + throw new Error( + `iss mismatch: expected '${expectedIssuer}', got '${issInRedirect}'` + ); + } + } + + // 7. Exchange code for token with PKCE code_verifier + const tokenResponse = await fetchFn(asMetadata.token_endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + grant_type: 'authorization_code', + code: authCode, + redirect_uri: 'http://localhost:3000/callback', + client_id: clientInfo.client_id, + code_verifier: codeVerifier + }).toString() + }); + + if (!tokenResponse.ok) { + const error = await tokenResponse.text(); + throw new Error(`Token request failed: ${tokenResponse.status} - ${error}`); + } + + return tokenResponse.json(); +} + +/** + * Creates a fetch wrapper that uses OAuth with iss parameter validation. + */ +function withOAuthIssValidation(baseUrl: string | URL): Middleware { + let tokens: OAuthTokens | undefined; + + return (next: FetchLike) => { + return async ( + input: string | URL, + init?: RequestInit + ): Promise => { + const makeRequest = async (): Promise => { + const headers = new Headers(init?.headers); + if (tokens) { + headers.set('Authorization', `Bearer ${tokens.access_token}`); + } + return next(input, { ...init, headers }); + }; + + let response = await makeRequest(); + + if (response.status === 401) { + const { resourceMetadataUrl } = extractWWWAuthenticateParams(response); + if (!resourceMetadataUrl) { + throw new Error('No resource_metadata in WWW-Authenticate'); + } + tokens = await oauthFlowWithIssValidation( + baseUrl, + resourceMetadataUrl, + next + ); + response = await makeRequest(); + } + + return response; + }; + }; +} + +export async function runClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-auth-client-iss-validation', version: '1.0.0' }, + { capabilities: {} } + ); + + const oauthFetch = withOAuthIssValidation(new URL(serverUrl))(fetch); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +runAsCli(runClient, import.meta.url, 'auth-test-iss-validation '); diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index 71c4582d..a9cf90cc 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -32,6 +32,7 @@ import { handle401 } from './helpers/withOAuthRetry.js'; import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider.js'; +import { runClient as issValidationClient } from './auth-test-iss-validation.js'; import { logger } from './helpers/logger.js'; /** @@ -270,7 +271,10 @@ registerScenarios( 'auth/resource-mismatch', // SEP-2207: Offline access / refresh token guidance (draft) 'auth/offline-access-scope', - 'auth/offline-access-not-supported' + 'auth/offline-access-not-supported', + // SEP-2468: ISS parameter - positive scenarios (standard client is fine) + 'auth/iss-supported', + 'auth/iss-not-advertised' ], runAuthClient ); @@ -342,6 +346,16 @@ async function runAuthMigrationClient(serverUrl: string): Promise { registerScenario('auth/authorization-server-migration', runAuthMigrationClient); +// SEP-2468: ISS parameter - rejection scenarios use iss-validating client +registerScenarios( + [ + 'auth/iss-supported-missing', + 'auth/iss-wrong-issuer', + 'auth/iss-unexpected' + ], + issValidationClient +); + // ============================================================================ // Elicitation defaults scenario // ============================================================================ diff --git a/src/scenarios/client/auth/helpers/createAuthServer.ts b/src/scenarios/client/auth/helpers/createAuthServer.ts index 00aa3dbc..5138e36a 100644 --- a/src/scenarios/client/auth/helpers/createAuthServer.ts +++ b/src/scenarios/client/auth/helpers/createAuthServer.ts @@ -43,6 +43,10 @@ export interface AuthServerOptions { disableDynamicRegistration?: boolean; /** PKCE code_challenge_methods_supported. Set to null to omit from metadata. Default: ['S256'] */ codeChallengeMethodsSupported?: string[] | null; + /** Advertise authorization_response_iss_parameter_supported in AS metadata. Default: true. Pass null to omit. */ + issParameterSupported?: boolean | null; + /** What iss value to include in authorization redirect. Default: 'correct'. */ + issInRedirect?: 'correct' | 'wrong' | 'omit'; tokenVerifier?: MockTokenVerifier; onTokenRequest?: (requestData: { scope?: string; @@ -86,6 +90,8 @@ export function createAuthServer( clientIdMetadataDocumentSupported, disableDynamicRegistration = false, codeChallengeMethodsSupported = ['S256'], + issParameterSupported = true, + issInRedirect = 'correct', tokenVerifier, onTokenRequest, onAuthorizationRequest, @@ -146,6 +152,9 @@ export function createAuthServer( ...(codeChallengeMethodsSupported !== null && { code_challenge_methods_supported: codeChallengeMethodsSupported }), + ...(issParameterSupported !== null && { + authorization_response_iss_parameter_supported: issParameterSupported + }), token_endpoint_auth_methods_supported: tokenEndpointAuthMethodsSupported, ...(tokenEndpointAuthSigningAlgValuesSupported && { token_endpoint_auth_signing_alg_values_supported: @@ -244,6 +253,13 @@ export function createAuthServer( redirectUrl.searchParams.set('state', state); } + // ISS: Include iss parameter in redirect if configured + if (issInRedirect === 'correct') { + redirectUrl.searchParams.set('iss', `${getAuthBaseUrl()}${routePrefix}`); + } else if (issInRedirect === 'wrong') { + redirectUrl.searchParams.set('iss', 'https://evil.example.com'); + } + res.redirect(redirectUrl.toString()); }); diff --git a/src/scenarios/client/auth/index.test.ts b/src/scenarios/client/auth/index.test.ts index c5d79811..cd09793a 100644 --- a/src/scenarios/client/auth/index.test.ts +++ b/src/scenarios/client/auth/index.test.ts @@ -16,6 +16,7 @@ import { runClient as noRetryLimitClient } from '../../../../examples/clients/ty import { runClient as noPkceClient } from '../../../../examples/clients/typescript/auth-test-no-pkce'; import { runClient as reuseCredsClient } from '../../../../examples/clients/typescript/auth-test-reuse-credentials'; import { runClient as noAppTypeClient } from '../../../../examples/clients/typescript/auth-test-no-application-type'; +import { runClient as noIssValidationClient } from '../../../../examples/clients/typescript/auth-test'; import { getHandler } from '../../../../examples/clients/typescript/everything-client'; import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger'; @@ -34,7 +35,11 @@ const allowClientErrorScenarios = new Set([ 'auth/resource-mismatch', // The post-migration retry path may surface as a client error after // re-registering; the SEP-2352 checks are evaluated in getChecks() - 'auth/authorization-server-migration' + 'auth/authorization-server-migration', + // Client is expected to error when iss validation fails + 'auth/iss-supported-missing', + 'auth/iss-wrong-issuer', + 'auth/iss-unexpected' ]); describe('Client Auth Scenarios', () => { @@ -174,4 +179,28 @@ describe('Negative tests', () => { ] }); }); + + test('client does not reject missing iss when server requires it', async () => { + const runner = new InlineClientRunner(noIssValidationClient); + await runClientAgainstScenario(runner, 'auth/iss-supported-missing', { + expectedFailureSlugs: ['sep-2468-client-reject-missing-iss'], + allowClientError: true + }); + }); + + test('client does not reject mismatched iss', async () => { + const runner = new InlineClientRunner(noIssValidationClient); + await runClientAgainstScenario(runner, 'auth/iss-wrong-issuer', { + expectedFailureSlugs: ['sep-2468-client-compare-iss-supported'], + allowClientError: true + }); + }); + + test('client does not reject unexpected iss', async () => { + const runner = new InlineClientRunner(noIssValidationClient); + await runClientAgainstScenario(runner, 'auth/iss-unexpected', { + expectedFailureSlugs: ['sep-2468-client-compare-iss-unadvertised'], + allowClientError: true + }); + }); }); diff --git a/src/scenarios/client/auth/index.ts b/src/scenarios/client/auth/index.ts index 85b3f3f9..92e87f69 100644 --- a/src/scenarios/client/auth/index.ts +++ b/src/scenarios/client/auth/index.ts @@ -29,6 +29,13 @@ import { OfflineAccessNotSupportedScenario } from './offline-access'; import { AuthorizationServerMigrationScenario } from './authorization-server-migration'; +import { + IssParameterSupportedScenario, + IssParameterNotAdvertisedScenario, + IssParameterSupportedMissingScenario, + IssParameterWrongIssuerScenario, + IssParameterUnexpectedScenario +} from './issuer-parameter'; // Auth scenarios (required for tier 1) export const authScenariosList: Scenario[] = [ @@ -63,5 +70,10 @@ export const draftScenariosList: Scenario[] = [ new ResourceMismatchScenario(), new OfflineAccessScopeScenario(), new OfflineAccessNotSupportedScenario(), - new AuthorizationServerMigrationScenario() + new AuthorizationServerMigrationScenario(), + new IssParameterSupportedScenario(), + new IssParameterNotAdvertisedScenario(), + new IssParameterSupportedMissingScenario(), + new IssParameterWrongIssuerScenario(), + new IssParameterUnexpectedScenario() ]; diff --git a/src/scenarios/client/auth/issuer-parameter.ts b/src/scenarios/client/auth/issuer-parameter.ts new file mode 100644 index 00000000..d34b73d0 --- /dev/null +++ b/src/scenarios/client/auth/issuer-parameter.ts @@ -0,0 +1,408 @@ +import type { Scenario, ConformanceCheck } from '../../../types.js'; +import { ScenarioUrls, DRAFT_PROTOCOL_VERSION } from '../../../types.js'; +import { createAuthServer } from './helpers/createAuthServer.js'; +import { createServer } from './helpers/createServer.js'; +import { ServerLifecycle } from './helpers/serverLifecycle.js'; +import { SpecReferences } from './spec-references.js'; +import { MockTokenVerifier } from './helpers/mockTokenVerifier.js'; + +const specRefs = [SpecReferences.RFC_9207_ISS_PARAMETER]; + +/** + * Scenario: ISS Parameter Supported (positive) + * + * Server advertises authorization_response_iss_parameter_supported: true and + * includes the correct iss value in the authorization redirect. A conformant + * client should validate iss and proceed normally. + */ +export class IssParameterSupportedScenario implements Scenario { + name = 'auth/iss-supported'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that client accepts authorization response when server advertises and sends correct iss parameter'; + + private authServer = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + private tokenRequestMade = false; + + async start(): Promise { + this.checks = []; + this.tokenRequestMade = false; + + const tokenVerifier = new MockTokenVerifier(this.checks, []); + + const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + tokenVerifier, + issParameterSupported: true, + issInRedirect: 'correct', + onTokenRequest: () => { + this.tokenRequestMade = true; + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); + await this.authServer.start(authApp); + + const app = createServer( + this.checks, + this.server.getUrl, + this.authServer.getUrl, + { requiredScopes: [], tokenVerifier } + ); + await this.server.start(app); + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + + this.checks.push({ + id: 'sep-2468-client-compare-iss-supported', + name: 'Client accepts matching iss when advertised', + description: this.tokenRequestMade + ? 'Client compared advertised iss against recorded issuer and proceeded to token exchange' + : 'Client did not proceed to token exchange after receiving a correct iss from a server that advertised support', + status: this.tokenRequestMade ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: specRefs, + details: { tokenRequestMade: this.tokenRequestMade } + }); + + return this.checks; + } +} + +/** + * Scenario: ISS Parameter Not Advertised (positive) + * + * Server does not advertise authorization_response_iss_parameter_supported and + * does not include iss in the redirect. A conformant client should proceed normally. + */ +export class IssParameterNotAdvertisedScenario implements Scenario { + name = 'auth/iss-not-advertised'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that client accepts authorization response when server does not advertise or send iss parameter'; + + private authServer = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + private tokenRequestMade = false; + + async start(): Promise { + this.checks = []; + this.tokenRequestMade = false; + + const tokenVerifier = new MockTokenVerifier(this.checks, []); + + const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + tokenVerifier, + issParameterSupported: null, + issInRedirect: 'omit', + onTokenRequest: () => { + this.tokenRequestMade = true; + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); + await this.authServer.start(authApp); + + const app = createServer( + this.checks, + this.server.getUrl, + this.authServer.getUrl, + { requiredScopes: [], tokenVerifier } + ); + await this.server.start(app); + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + + this.checks.push({ + id: 'sep-2468-client-proceed-no-iss', + name: 'Client proceeds when iss absent and not advertised', + description: this.tokenRequestMade + ? 'Client proceeded to token exchange when neither metadata advertised iss support nor redirect contained iss' + : 'Client did not proceed to token exchange — should proceed when iss is absent and not advertised', + status: this.tokenRequestMade ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: specRefs, + details: { tokenRequestMade: this.tokenRequestMade } + }); + + return this.checks; + } +} + +/** + * Scenario: ISS Parameter Advertised but Missing from Redirect (client must reject) + * + * Server advertises authorization_response_iss_parameter_supported: true but + * omits iss from the redirect. A conformant client MUST reject this response. + */ +export class IssParameterSupportedMissingScenario implements Scenario { + name = 'auth/iss-supported-missing'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that client rejects authorization response when server advertised iss support but omitted iss from redirect'; + allowClientError = true; + + private authServer = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + private authReached = false; + private tokenRequestMade = false; + + async start(): Promise { + this.checks = []; + this.authReached = false; + this.tokenRequestMade = false; + + const tokenVerifier = new MockTokenVerifier(this.checks, []); + + const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + tokenVerifier, + issParameterSupported: true, + issInRedirect: 'omit', // advertise support but don't send iss + onAuthorizationRequest: () => { + this.authReached = true; + }, + onTokenRequest: () => { + this.tokenRequestMade = true; + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); + await this.authServer.start(authApp); + + const app = createServer( + this.checks, + this.server.getUrl, + this.authServer.getUrl, + { requiredScopes: [], tokenVerifier } + ); + await this.server.start(app); + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + + if ( + !this.checks.some((c) => c.id === 'sep-2468-client-reject-missing-iss') + ) { + const correctlyRejected = this.authReached && !this.tokenRequestMade; + this.checks.push({ + id: 'sep-2468-client-reject-missing-iss', + name: 'Client rejects missing iss when required', + description: correctlyRejected + ? 'Client correctly rejected authorization response missing required iss parameter' + : 'Client MUST reject authorization response when server advertised iss support but iss is absent from redirect', + status: correctlyRejected ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: specRefs, + details: { + serverAdvertisedSupport: true, + issSentInRedirect: false, + authReached: this.authReached, + tokenRequestMade: this.tokenRequestMade + } + }); + } + + return this.checks; + } +} + +/** + * Scenario: ISS Parameter Has Wrong Value (client must reject) + * + * Server advertises authorization_response_iss_parameter_supported: true and + * includes an iss value that does not match the server's actual issuer. A + * conformant client MUST reject this response. + */ +export class IssParameterWrongIssuerScenario implements Scenario { + name = 'auth/iss-wrong-issuer'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that client rejects authorization response when iss does not match the authorization server issuer'; + allowClientError = true; + + private authServer = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + private authReached = false; + private tokenRequestMade = false; + + async start(): Promise { + this.checks = []; + this.authReached = false; + this.tokenRequestMade = false; + + const tokenVerifier = new MockTokenVerifier(this.checks, []); + + const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + tokenVerifier, + issParameterSupported: true, + issInRedirect: 'wrong', // send iss that doesn't match metadata issuer + onAuthorizationRequest: () => { + this.authReached = true; + }, + onTokenRequest: () => { + this.tokenRequestMade = true; + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); + await this.authServer.start(authApp); + + const app = createServer( + this.checks, + this.server.getUrl, + this.authServer.getUrl, + { requiredScopes: [], tokenVerifier } + ); + await this.server.start(app); + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + + if ( + !this.checks.some((c) => c.id === 'sep-2468-client-compare-iss-supported') + ) { + const correctlyRejected = this.authReached && !this.tokenRequestMade; + this.checks.push({ + id: 'sep-2468-client-compare-iss-supported', + name: 'Client rejects mismatched iss', + description: correctlyRejected + ? 'Client correctly rejected authorization response with mismatched iss parameter' + : 'Client MUST reject authorization response when iss does not match the authorization server issuer', + status: correctlyRejected ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: specRefs, + details: { + serverAdvertisedSupport: true, + issSentInRedirect: 'https://evil.example.com', + authReached: this.authReached, + tokenRequestMade: this.tokenRequestMade + } + }); + } + + return this.checks; + } +} + +/** + * Scenario: ISS Parameter Sent but Not Advertised, Mismatched (client must reject) + * + * Server does not advertise authorization_response_iss_parameter_supported but + * includes a mismatched iss value in the redirect. Per the SEP-2468 spec table + * row 3, a conformant client MUST compare a present iss against the recorded + * issuer regardless of metadata advertisement, and reject on mismatch. + */ +export class IssParameterUnexpectedScenario implements Scenario { + name = 'auth/iss-unexpected'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + description = + 'Tests that client compares iss against recorded issuer even when not advertised, and rejects on mismatch'; + allowClientError = true; + + private authServer = new ServerLifecycle(); + private server = new ServerLifecycle(); + private checks: ConformanceCheck[] = []; + private authReached = false; + private tokenRequestMade = false; + + async start(): Promise { + this.checks = []; + this.authReached = false; + this.tokenRequestMade = false; + + const tokenVerifier = new MockTokenVerifier(this.checks, []); + + const authApp = createAuthServer(this.checks, this.authServer.getUrl, { + tokenVerifier, + issParameterSupported: null, + issInRedirect: 'wrong', // send mismatched iss without advertising support + onAuthorizationRequest: () => { + this.authReached = true; + }, + onTokenRequest: () => { + this.tokenRequestMade = true; + return { token: `test-token-${Date.now()}`, scopes: [] }; + } + }); + await this.authServer.start(authApp); + + const app = createServer( + this.checks, + this.server.getUrl, + this.authServer.getUrl, + { requiredScopes: [], tokenVerifier } + ); + await this.server.start(app); + + return { serverUrl: `${this.server.getUrl()}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + + if ( + !this.checks.some( + (c) => c.id === 'sep-2468-client-compare-iss-unadvertised' + ) + ) { + const correctlyRejected = this.authReached && !this.tokenRequestMade; + this.checks.push({ + id: 'sep-2468-client-compare-iss-unadvertised', + name: 'Client compares unadvertised iss and rejects mismatch', + description: correctlyRejected + ? 'Client correctly compared unadvertised iss against recorded issuer and rejected the mismatch' + : 'Client MUST compare a present iss against the recorded issuer regardless of metadata advertisement, and reject on mismatch', + status: correctlyRejected ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: specRefs, + details: { + serverAdvertisedSupport: false, + issSentInRedirect: 'https://evil.example.com', + authReached: this.authReached, + tokenRequestMade: this.tokenRequestMade + } + }); + } + + return this.checks; + } +} diff --git a/src/scenarios/client/auth/spec-references.ts b/src/scenarios/client/auth/spec-references.ts index 768dd65f..908a04eb 100644 --- a/src/scenarios/client/auth/spec-references.ts +++ b/src/scenarios/client/auth/spec-references.ts @@ -89,6 +89,11 @@ export const SpecReferences: { [key: string]: SpecReference } = { id: 'MCP-PKCE-requirement', url: 'https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization#authorization-code-protection' }, + // TODO: Update to MCP spec URL once SEP-2468 (modelcontextprotocol/modelcontextprotocol#2468) is merged + RFC_9207_ISS_PARAMETER: { + id: 'RFC-9207-iss-parameter', + url: 'https://www.rfc-editor.org/rfc/rfc9207.html#section-2' + }, RFC_8693_TOKEN_EXCHANGE: { id: 'RFC-8693-Token-Exchange', url: 'https://datatracker.ietf.org/doc/html/rfc8693' diff --git a/src/seps/sep-2468.yaml b/src/seps/sep-2468.yaml new file mode 100644 index 00000000..6558ae6e --- /dev/null +++ b/src/seps/sep-2468.yaml @@ -0,0 +1,23 @@ +sep: 2468 +spec_url: https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-response-validation +requirements: + - check: sep-2468-client-validate-metadata-issuer + text: 'After retrieving a metadata document, MCP clients MUST validate it as required by RFC8414 Section 3.3 or OpenID Connect Discovery Section 4.3: the issuer value in the document MUST be identical to the issuer identifier used to construct the well-known URL. If they differ, the client MUST NOT use the metadata.' + url: https://modelcontextprotocol.io/specification/draft/basic/authorization#authorization-server-metadata-discovery + - check: sep-2468-as-include-iss + text: 'MCP authorization servers SHOULD include the iss parameter in authorization responses, including error responses, as defined in RFC9207 Section 2.' + - check: sep-2468-as-advertise-iss-supported + text: 'Authorization servers that include the iss parameter MUST advertise this by setting authorization_response_iss_parameter_supported to true in their metadata (RFC9207 Section 2.3).' + - check: sep-2468-client-compare-iss-supported + text: 'On receiving the authorization response, MCP clients MUST apply the validation in RFC9207 Section 2.4 before transmitting the authorization code to any token endpoint: authorization_response_iss_parameter_supported true, iss present -> Compare to the recorded issuer using simple string comparison / Before redirecting the user-agent, the client MUST record the issuer value from the selected authorization server validated metadata document and associate it with the same per-request record used to store the PKCE code verifier.' + - check: sep-2468-client-reject-missing-iss + text: 'On receiving the authorization response, MCP clients MUST apply the validation in RFC9207 Section 2.4 before transmitting the authorization code to any token endpoint: authorization_response_iss_parameter_supported true, iss absent -> Reject the response.' + - check: sep-2468-client-compare-iss-unadvertised + text: 'On receiving the authorization response, MCP clients MUST apply the validation in RFC9207 Section 2.4 before transmitting the authorization code to any token endpoint: authorization_response_iss_parameter_supported false or absent, iss present -> Compare to the recorded issuer using simple string comparison.' + - check: sep-2468-client-proceed-no-iss + text: 'On receiving the authorization response, MCP clients MUST apply the validation in RFC9207 Section 2.4 before transmitting the authorization code to any token endpoint: authorization_response_iss_parameter_supported false or absent, iss absent -> Proceed.' + - check: sep-2468-client-no-normalization + text: 'After decoding the iss value from the application/x-www-form-urlencoded response per RFC 9207 Section 2.4, clients MUST NOT apply scheme or host case folding, default-port elision, trailing-slash, or percent-encoding normalization (RFC 3986 Sections 6.2.2-6.2.3) before comparison.' + + - text: 'This validation applies equally to error responses - on mismatch the client MUST NOT act on or display error, error_description, or error_uri.' + excluded: 'display is UI-facing; act-on has no protocol-observable signal beyond the existing reject-on-mismatch checks'