feat: sdk subcommand to run local conformance against any SDK ref

.gitignore

@@ -5,3 +5,4 @@ dist/
 .vscode/
 .idea/
 .claude/settings.local.json
+.sdk-under-test/

AGENTS.md

@@ -52,6 +52,7 @@ Keep scenarios separate when they're genuinely independent features or when they
 - **Same `id` for SUCCESS and FAIL.** A check should use one slug and flip `status` + `errorMessage`, not branch into `foo-success` vs `foo-failure` slugs.
 - **Optimize for Ctrl+F on the slug.** Repetitive check blocks are fine — easier to find the failing one than to unwind a clever helper.
 - Reuse `ConformanceCheck` and other types from `src/types.ts` rather than defining parallel shapes.
+- **Don't reimplement the runner.** New subcommands that need to "select scenarios → run them → print summary → compute exit code" must go through the existing `client` / `server` commands (subprocess via `process.execPath` like `tier-check` and `sdk` do) or call shared helpers — never a parallel suite-map / summary loop.
 - Include `specReferences` pointing to the relevant spec section.
 - **Severity follows the spec keyword:** MUST / MUST NOT → `FAILURE`; SHOULD / SHOULD NOT → `WARNING`. (CI treats WARNING as a failure, so Tier-1 SDKs still need to satisfy SHOULDs — see #245.)
 

README.md

@@ -210,6 +210,40 @@ Run `npx @modelcontextprotocol/conformance list --server` to see all available s
 - **resources-\*** - Resource management scenarios
 - **prompts-\*** - Prompt management scenarios
 
+## Running Against an SDK at a Specific Ref
+
+The `sdk` subcommand clones an SDK repository at a given ref, builds it, and runs the **local** conformance build against it. This is the inner-loop tool for scenario authors and the basis for cross-SDK CI. Examples below use `npm start --` so they run from source — no `npm run build` between edits.
+
+```bash
+# Clone and run everything against typescript-sdk@main
+npm start -- sdk typescript-sdk@main
+
+# Against a specific tag, SHA, or branch
+npm start -- sdk typescript-sdk@v1.29.0
+npm start -- sdk typescript-sdk@abc123f
+npm start -- sdk python-sdk@some-feature-branch
+
+# Use an existing local checkout (no clone, no fetch)
+npm start -- sdk --path ../typescript-sdk --skip-build
+
+# Narrow to one mode / scenario / suite
+npm start -- sdk --path ../typescript-sdk --mode server --scenario server-initialize
+npm start -- sdk typescript-sdk@main --mode client --suite auth
+```
+
+Build/run commands for each official SDK are looked up by name from [`src/sdk-runner/known-sdks.ts`](src/sdk-runner/known-sdks.ts) — no config file is required in the SDK repo. Resolution order is **CLI flag > `conformance.config.yaml` in the SDK checkout (optional override) > built-in entry**, so any field can be overridden on the command line for refs that diverge from the built-in:
+
+```bash
+npm start -- sdk owner/go-sdk@some-branch \
+  --mode client \
+  --build-cmd 'go build -tags mcp_go_client_oauth -o ./.conformance-client ./conformance/everything-client' \
+  --client-cmd './.conformance-client'
+```
+
+To add a new SDK to the matrix, add an entry to `KNOWN_SDKS`.
+
+Clones are cached under `.sdk-under-test/` and reused (fetched) on subsequent runs.
+
 ## SDK Tier Assessment
 
 The `tier-check` subcommand evaluates an MCP SDK repository against [SEP-1730](https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1730) (the SDK Tiering System):

src/index.ts

@@ -46,6 +46,7 @@ import {
 } from './expected-failures';
 import { createTierCheckCommand } from './tier-check';
 import { createNewSepCommand } from './new-sep';
+import { createSdkCommand } from './sdk-runner';
 import packageJson from '../package.json';
 
 // Note on naming: `command` refers to which CLI command is calling this.
@@ -544,6 +545,9 @@ program.addCommand(createTierCheckCommand());
 // New SEP scaffolding command
 program.addCommand(createNewSepCommand());
 
+// SDK command - run local conformance against an SDK at a specific ref
+program.addCommand(createSdkCommand());
+
 // List scenarios command
 program
   .command('list')

src/sdk-runner/checkout.ts

@@ -0,0 +1,109 @@
+import { spawn } from 'child_process';
+import { promises as fs } from 'fs';
+import path from 'path';
+
+export interface SdkSpec {
+  name: string;
+  ref: string;
+}
+
+const DEFAULT_ORG = 'modelcontextprotocol';
+
+export function parseSdkSpec(spec: string): SdkSpec {
+  const at = spec.lastIndexOf('@');
+  if (at <= 0) {
+    return { name: spec, ref: 'main' };
+  }
+  return { name: spec.slice(0, at), ref: spec.slice(at + 1) };
+}
+
+function repoUrl(name: string): string {
+  if (name.includes('/')) {
+    return `https://github.com/${name}.git`;
+  }
+  return `https://github.com/${DEFAULT_ORG}/${name}.git`;
+}
+
+async function git(
+  args: string[],
+  cwd: string
+): Promise<{ stdout: string; stderr: string }> {
+  const cmd = 'git';
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (d) => (stdout += d.toString()));
+    child.stderr.on('data', (d) => (stderr += d.toString()));
+    child.on('error', reject);
+    child.on('close', (code) => {
+      if (code === 0) {
+        resolve({ stdout, stderr });
+      } else {
+        reject(
+          new Error(
+            `${cmd} ${args.join(' ')} exited with ${code}\n${stderr || stdout}`
+          )
+        );
+      }
+    });
+  });
+}
+
+async function dirExists(dir: string): Promise<boolean> {
+  try {
+    const stat = await fs.stat(dir);
+    return stat.isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Ensure an SDK is checked out at the requested ref under cacheDir.
+ * Clones on first use; on subsequent calls fetches and resets to the ref.
+ * Returns the absolute path to the checkout.
+ */
+export async function ensureCheckout(
+  spec: SdkSpec,
+  cacheDir: string
+): Promise<string> {
+  await fs.mkdir(cacheDir, { recursive: true });
+  const safeName = spec.name.replace('/', '__');
+  const dir = path.resolve(cacheDir, safeName);
+
+  if (await dirExists(path.join(dir, '.git'))) {
+    console.error(`[sdk] Fetching ${spec.name} (cached at ${dir})`);
+    await git(['fetch', '--tags', 'origin'], dir);
+  } else {
+    console.error(`[sdk] Cloning ${repoUrl(spec.name)} -> ${dir}`);
+    await git(['clone', repoUrl(spec.name), dir], cacheDir);
+  }
+
+  // Try the ref as a remote branch first, then fall back to a local-resolvable
+  // ref (tag or SHA).
+  const candidates = [`origin/${spec.ref}`, spec.ref];
+  let resolved: string | undefined;
+  for (const candidate of candidates) {
+    try {
+      await git(['rev-parse', '--verify', `${candidate}^{commit}`], dir);
+      resolved = candidate;
+      break;
+    } catch {
+      // rev-parse failure means this candidate doesn't exist; try the next form
+    }
+  }
+  if (!resolved) {
+    throw new Error(
+      `Ref '${spec.ref}' not found in ${spec.name} (tried ${candidates.join(', ')})`
+    );
+  }
+
+  console.error(`[sdk] Checking out ${spec.name}@${spec.ref} (${resolved})`);
+  await git(['checkout', '--detach', resolved], dir);
+
+  const { stdout } = await git(['rev-parse', '--short', 'HEAD'], dir);
+  console.error(`[sdk] HEAD is ${stdout.trim()}`);
+
+  return dir;
+}

src/sdk-runner/config.ts

@@ -0,0 +1,44 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import { parse as parseYaml } from 'yaml';
+import { z } from 'zod';
+
+export const SdkConfigSchema = z.object({
+  build: z.string().optional(),
+  client: z
+    .object({
+      command: z.string()
+    })
+    .optional(),
+  server: z
+    .object({
+      command: z.string(),
+      url: z.string().url(),
+      readyTimeoutMs: z.number().int().positive().optional()
+    })
+    .optional(),
+  expectedFailures: z.string().optional()
+});
+
+export type SdkConfig = z.infer<typeof SdkConfigSchema>;
+
+const CONFIG_FILENAMES = [
+  'conformance.config.yaml',
+  'conformance.config.yml',
+  'conformance.config.json'
+];
+
+export async function loadSdkConfig(dir: string): Promise<SdkConfig | null> {
+  for (const name of CONFIG_FILENAMES) {
+    const filePath = path.join(dir, name);
+    let raw: string;
+    try {
+      raw = await fs.readFile(filePath, 'utf-8');
+    } catch {
+      continue;
+    }
+    const parsed = name.endsWith('.json') ? JSON.parse(raw) : parseYaml(raw);
+    return SdkConfigSchema.parse(parsed);
+  }
+  return null;
+}