diff --git a/.changeset/mcp-exposed-tools.md b/.changeset/mcp-exposed-tools.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/mcp-exposed-tools.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/server/src/mcp/exposed-tools.ts b/server/src/mcp/exposed-tools.ts new file mode 100644 index 0000000000..834d249746 --- /dev/null +++ b/server/src/mcp/exposed-tools.ts @@ -0,0 +1,210 @@ +/** + * Exposed MCP Tools + * + * Internal Addie tools promoted to first-class MCP tools on the /mcp endpoint, + * callable directly by external MCP clients (Claude Code, Claude Desktop, etc.). + * + * Three categories: + * 1. Evaluation — agent testing and compliance (requires auth) + * 2. Agent context — save/list/remove agent credentials (requires auth) + * 3. Validation — schema and adagents.json validation (stateless, no auth) + */ + +import { createLogger } from '../logger.js'; +import { MEMBER_TOOLS, createMemberToolHandlers } from '../addie/mcp/member-tools.js'; +import { SCHEMA_TOOLS, createSchemaToolHandlers } from '../addie/mcp/schema-tools.js'; +import { PROPERTY_TOOLS, createPropertyToolHandlers } from '../addie/mcp/property-tools.js'; +import type { MemberContext } from '../addie/member-context.js'; +import type { MCPAuthContext } from './auth.js'; + +const logger = createLogger('mcp-exposed-tools'); + +// ── Tool name sets ────────────────────────────────────────────────── + +/** Agent evaluation tools (require auth for saved-agent credential lookup). */ +const EVAL_TOOL_NAMES = [ + 'probe_adcp_agent', + 'evaluate_agent_quality', + 'test_rfp_response', + 'test_io_execution', +] as const; + +/** Agent context management tools (require auth). */ +const AGENT_CONTEXT_TOOL_NAMES = [ + 'save_agent', + 'list_saved_agents', + 'remove_saved_agent', +] as const; + +/** Schema validation tools (stateless, no auth required). */ +const SCHEMA_TOOL_NAMES = [ + 'validate_json', + 'get_schema', +] as const; + +/** Property validation tools (stateless, no auth required). */ +const PROPERTY_TOOL_NAMES = [ + 'validate_adagents', +] as const; + +// ── Startup validation ────────────────────────────────────────────── +// Fail at import time if any exposed tool name was renamed or removed upstream. + +const memberToolNames = new Set(MEMBER_TOOLS.map((t) => t.name)); +for (const name of [...EVAL_TOOL_NAMES, ...AGENT_CONTEXT_TOOL_NAMES]) { + if (!memberToolNames.has(name)) { + throw new Error(`Exposed tool "${name}" not found in MEMBER_TOOLS — was it renamed or removed?`); + } +} + +const schemaToolNames = new Set(SCHEMA_TOOLS.map((t) => t.name)); +for (const name of SCHEMA_TOOL_NAMES) { + if (!schemaToolNames.has(name)) { + throw new Error(`Exposed tool "${name}" not found in SCHEMA_TOOLS — was it renamed or removed?`); + } +} + +const propertyToolNames = new Set(PROPERTY_TOOLS.map((t) => t.name)); +for (const name of PROPERTY_TOOL_NAMES) { + if (!propertyToolNames.has(name)) { + throw new Error(`Exposed tool "${name}" not found in PROPERTY_TOOLS — was it renamed or removed?`); + } +} + +// ── Tool definitions (MCP format) ─────────────────────────────────── + +// usage_hints are intentionally excluded — they're for Addie's internal router, +// not for external MCP clients. +function toMCPFormat(tool: { name: string; description: string; input_schema: object }) { + return { name: tool.name, description: tool.description, inputSchema: tool.input_schema }; +} + +/** Evaluation tool definitions in MCP format. */ +export const EVAL_TOOL_DEFINITIONS = MEMBER_TOOLS + .filter((t) => (EVAL_TOOL_NAMES as readonly string[]).includes(t.name)) + .map(toMCPFormat); + +/** Agent context tool definitions in MCP format. */ +export const AGENT_CONTEXT_TOOL_DEFINITIONS = MEMBER_TOOLS + .filter((t) => (AGENT_CONTEXT_TOOL_NAMES as readonly string[]).includes(t.name)) + .map(toMCPFormat); + +/** Schema validation tool definitions in MCP format. */ +export const SCHEMA_TOOL_DEFINITIONS = SCHEMA_TOOLS + .filter((t) => (SCHEMA_TOOL_NAMES as readonly string[]).includes(t.name)) + .map(toMCPFormat); + +/** Property validation tool definitions in MCP format. */ +export const PROPERTY_TOOL_DEFINITIONS = PROPERTY_TOOLS + .filter((t) => (PROPERTY_TOOL_NAMES as readonly string[]).includes(t.name)) + .map(toMCPFormat); + +/** All exposed tool definitions combined. */ +export const ALL_EXPOSED_TOOL_DEFINITIONS = [ + ...EVAL_TOOL_DEFINITIONS, + ...AGENT_CONTEXT_TOOL_DEFINITIONS, + ...SCHEMA_TOOL_DEFINITIONS, + ...PROPERTY_TOOL_DEFINITIONS, +]; + +// ── Auth bridging ─────────────────────────────────────────────────── + +/** + * Build a minimal MemberContext from MCP auth claims. + * + * Identity is verified via OAuth JWT, but membership/subscription status + * is not resolved from the database. Tools gate on orgId presence for + * credential lookup rather than membership status. + */ +function mcpAuthToMemberContext(auth: MCPAuthContext): MemberContext { + return { + is_mapped: true, + is_member: false, + workos_user: { + workos_user_id: auth.sub, + email: auth.email || '', + }, + organization: auth.orgId + ? { + workos_organization_id: auth.orgId, + name: '', + subscription_status: null, + is_personal: false, + } + : undefined, + } as MemberContext; +} + +// ── Handler factories ─────────────────────────────────────────────── + +/** + * Create a handler for a member tool (eval or agent context) that bridges + * MCPAuthContext to MemberContext. Requires authentication — anonymous + * callers receive an error with isError: true. + */ +export function createMemberToolHandler(toolName: string) { + return async ( + args: Record, + authContext?: MCPAuthContext, + ): Promise<{ content: Array<{ type: string; text: string }>; isError?: boolean }> => { + if (!authContext || authContext.sub === 'anonymous') { + return { + content: [{ type: 'text', text: 'Authentication required. Connect with OAuth to use this tool.' }], + isError: true, + }; + } + + const memberContext = mcpAuthToMemberContext(authContext); + const handlers = createMemberToolHandlers(memberContext); + const handler = handlers.get(toolName); + + if (!handler) { + logger.error({ toolName }, 'Member tool handler not found'); + return { + content: [{ type: 'text', text: JSON.stringify({ error: `Unknown tool: ${toolName}` }) }], + isError: true, + }; + } + + const result = await handler(args); + return { content: [{ type: 'text', text: result }] }; + }; +} + +/** + * Create handlers for stateless tools (schema and property validation). + * These are created once at startup since they don't need per-request auth. + */ +export function createStatelessToolHandlers(): Map< + string, + (args: Record) => Promise<{ content: Array<{ type: string; text: string }> }> +> { + const result = new Map< + string, + (args: Record) => Promise<{ content: Array<{ type: string; text: string }> }> + >(); + + const schemaHandlers = createSchemaToolHandlers(); + for (const name of SCHEMA_TOOL_NAMES) { + const handler = schemaHandlers.get(name); + if (handler) { + result.set(name, async (args) => { + const text = await handler(args); + return { content: [{ type: 'text', text }] }; + }); + } + } + + const propertyHandlers = createPropertyToolHandlers(); + for (const name of PROPERTY_TOOL_NAMES) { + const handler = propertyHandlers.get(name); + if (handler) { + result.set(name, async (args) => { + const text = await handler(args); + return { content: [{ type: 'text', text }] }; + }); + } + } + + return result; +} diff --git a/server/src/mcp/server.ts b/server/src/mcp/server.ts index 97ba2493fc..1ba04e2017 100644 --- a/server/src/mcp/server.ts +++ b/server/src/mcp/server.ts @@ -4,6 +4,9 @@ * Public MCP interface exposing: * - chat_with_addie: Conversational AI (wraps knowledge + directory tools internally) * - Directory tools: Programmatic lookup (list_members, list_agents, etc.) + * - Evaluation tools: Agent testing (probe, compliance, RFP response, IO execution) + * - Agent context tools: Save/list/remove agent credentials + * - Validation tools: Schema validation and adagents.json checking * * Knowledge and billing tools are NOT exposed directly - they're available * through chat_with_addie for conversational access, or internal Slack use only. @@ -38,6 +41,15 @@ import { MCPToolHandler, TOOL_DEFINITIONS, RESOURCE_DEFINITIONS } from '../mcp-t // Chat tool - conversational AI wrapper (has knowledge + directory tools internally) import { CHAT_TOOL, createChatToolHandler } from './chat-tool.js'; +// Exposed tools - internal tools promoted to first-class MCP tools +import { + ALL_EXPOSED_TOOL_DEFINITIONS, + EVAL_TOOL_DEFINITIONS, + AGENT_CONTEXT_TOOL_DEFINITIONS, + createMemberToolHandler, + createStatelessToolHandlers, +} from './exposed-tools.js'; + const logger = createLogger('mcp-server'); /** @@ -54,31 +66,37 @@ function convertToMCPTool(tool: AddieTool) { /** * All tools available in the unified MCP server * - * Only exposes: + * Exposes: * - chat_with_addie: Conversational wrapper (uses knowledge + directory tools internally) * - Directory tools: Programmatic member/agent/publisher lookup + * - Evaluation tools: Agent testing (probe, compliance, RFP, IO execution) + * - Agent context tools: Save/list/remove agent credentials + * - Validation tools: Schema validation and adagents.json checking * * Knowledge and billing tools are NOT exposed - use chat_with_addie instead. */ export function getAllTools() { const chatTool = convertToMCPTool(CHAT_TOOL); - - // Directory tools are already in MCP format const directoryTools = TOOL_DEFINITIONS; + const exposedTools = ALL_EXPOSED_TOOL_DEFINITIONS; return { directory: directoryTools, + exposed: exposedTools, chat: chatTool, - all: [chatTool, ...directoryTools], + all: [chatTool, ...directoryTools, ...exposedTools], }; } /** * Create all tool handlers * - * Only creates handlers for publicly exposed tools: + * Creates handlers for publicly exposed tools: * - chat_with_addie * - Directory tools (list_members, list_agents, etc.) + * - Evaluation tools (probe, compliance, RFP, IO execution) + * - Agent context tools (save_agent, list_saved_agents, remove_saved_agent) + * - Validation tools (validate_json, get_schema, validate_adagents) */ function createAllHandlers() { const handlers = new Map, authContext?: MCPAuthContext) => Promise>(); @@ -100,6 +118,18 @@ function createAllHandlers() { }); } + // Member tools (eval + agent context) — need per-request auth bridging + const memberToolDefs = [...EVAL_TOOL_DEFINITIONS, ...AGENT_CONTEXT_TOOL_DEFINITIONS]; + for (const tool of memberToolDefs) { + handlers.set(tool.name, createMemberToolHandler(tool.name)); + } + + // Stateless tools (schema + property validation) — created once + const statelessHandlers = createStatelessToolHandlers(); + for (const [name, handler] of statelessHandlers) { + handlers.set(name, async (args) => handler(args)); + } + return { handlers, directoryHandler }; } @@ -124,6 +154,9 @@ function getHandlers() { * This server exposes Addie capabilities via MCP: * - chat_with_addie: Conversational AI with knowledge + directory access * - Directory tools: Programmatic lookup of members, agents, publishers + * - Evaluation tools: Agent testing and compliance checking + * - Agent context tools: Credential management for agent testing + * - Validation tools: Schema and adagents.json validation */ export function createUnifiedMCPServer(authContext?: MCPAuthContext): Server { const server = new Server( diff --git a/server/tests/integration/mcp-protocol.test.ts b/server/tests/integration/mcp-protocol.test.ts index 728fe17e3b..5c3c5cb865 100644 --- a/server/tests/integration/mcp-protocol.test.ts +++ b/server/tests/integration/mcp-protocol.test.ts @@ -62,12 +62,17 @@ vi.mock('../../src/db/member-db.js', () => { }); // Mock rate limiter to disable validation in tests -vi.mock('../../src/middleware/rate-limit.js', () => ({ - apiRateLimiter: (req: any, res: any, next: any) => next(), - authRateLimiter: (req: any, res: any, next: any) => next(), - webhookRateLimiter: (req: any, res: any, next: any) => next(), - invitationRateLimiter: (req: any, res: any, next: any) => next(), -})); +vi.mock('../../src/middleware/rate-limit.js', async (importOriginal) => { + const passthrough = (req: any, res: any, next: any) => next(); + return { + ...(await importOriginal() as Record), + apiRateLimiter: passthrough, + authRateLimiter: passthrough, + webhookRateLimiter: passthrough, + invitationRateLimiter: passthrough, + orgCreationRateLimiter: passthrough, + }; +}); describe('MCP Protocol Compliance', () => { let server: HTTPServer; @@ -119,7 +124,7 @@ describe('MCP Protocol Compliance', () => { }); expect(response.body.result.tools).toBeInstanceOf(Array); - expect(response.body.result.tools).toHaveLength(22); + expect(response.body.result.tools).toHaveLength(32); }); it('each tool has name, description, inputSchema', async () => { @@ -159,6 +164,97 @@ describe('MCP Protocol Compliance', () => { }); }); + describe('POST /mcp - evaluation tools in tools/list', () => { + const EVAL_TOOLS = [ + 'probe_adcp_agent', + 'evaluate_agent_quality', + 'test_rfp_response', + 'test_io_execution', + ]; + + it('includes all evaluation tools', async () => { + const response = await request(app) + .post('/mcp') + .send({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list' + }); + + const toolNames = response.body.result.tools.map((t: any) => t.name); + for (const name of EVAL_TOOLS) { + expect(toolNames).toContain(name); + } + }); + + it('evaluate_agent_quality requires agent_url', async () => { + const response = await request(app) + .post('/mcp') + .send({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list' + }); + + const tool = response.body.result.tools.find((t: any) => t.name === 'evaluate_agent_quality'); + expect(tool.inputSchema.required).toContain('agent_url'); + }); + + it('test_rfp_response requires agent_url and rfp', async () => { + const response = await request(app) + .post('/mcp') + .send({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list' + }); + + const tool = response.body.result.tools.find((t: any) => t.name === 'test_rfp_response'); + expect(tool.inputSchema.required).toContain('agent_url'); + expect(tool.inputSchema.required).toContain('rfp'); + }); + + it('test_io_execution requires agent_url and line_items', async () => { + const response = await request(app) + .post('/mcp') + .send({ + jsonrpc: '2.0', + id: 1, + method: 'tools/list' + }); + + const tool = response.body.result.tools.find((t: any) => t.name === 'test_io_execution'); + expect(tool.inputSchema.required).toContain('agent_url'); + expect(tool.inputSchema.required).toContain('line_items'); + }); + }); + + describe('POST /mcp - agent context tools in tools/list', () => { + it('includes save_agent, list_saved_agents, remove_saved_agent', async () => { + const response = await request(app) + .post('/mcp') + .send({ jsonrpc: '2.0', id: 1, method: 'tools/list' }); + + const toolNames = response.body.result.tools.map((t: any) => t.name); + expect(toolNames).toContain('save_agent'); + expect(toolNames).toContain('list_saved_agents'); + expect(toolNames).toContain('remove_saved_agent'); + }); + }); + + describe('POST /mcp - validation tools in tools/list', () => { + it('includes validate_json, get_schema, validate_adagents', async () => { + const response = await request(app) + .post('/mcp') + .send({ jsonrpc: '2.0', id: 1, method: 'tools/list' }); + + const toolNames = response.body.result.tools.map((t: any) => t.name); + expect(toolNames).toContain('validate_json'); + expect(toolNames).toContain('get_schema'); + expect(toolNames).toContain('validate_adagents'); + }); + }); + describe('POST /mcp - tools/call: list_agents', () => { it('returns structured content with type: "resource"', async () => { const response = await request(app) diff --git a/server/tests/unit/mcp-eval-tools.test.ts b/server/tests/unit/mcp-eval-tools.test.ts new file mode 100644 index 0000000000..d395562625 --- /dev/null +++ b/server/tests/unit/mcp-eval-tools.test.ts @@ -0,0 +1,184 @@ +/** + * Tests for tools exposed on the /mcp endpoint. + * + * Verifies that: + * - Tool definitions are correctly extracted from internal tool arrays + * - Each tool has proper MCP-format schema + * - Handler factories create callable handlers + * - Auth-required tools reject anonymous callers + */ +import { describe, it, expect } from 'vitest'; +import { + EVAL_TOOL_DEFINITIONS, + AGENT_CONTEXT_TOOL_DEFINITIONS, + SCHEMA_TOOL_DEFINITIONS, + PROPERTY_TOOL_DEFINITIONS, + ALL_EXPOSED_TOOL_DEFINITIONS, + createMemberToolHandler, + createStatelessToolHandlers, +} from '../../src/mcp/exposed-tools.js'; + +describe('EVAL_TOOL_DEFINITIONS', () => { + const EXPECTED = [ + 'probe_adcp_agent', + 'evaluate_agent_quality', + 'test_rfp_response', + 'test_io_execution', + ]; + + it('exports exactly the 4 evaluation tools', () => { + const names = EVAL_TOOL_DEFINITIONS.map((t) => t.name); + expect(names).toEqual(expect.arrayContaining(EXPECTED)); + expect(names).toHaveLength(EXPECTED.length); + }); + + it('each tool has name, description, and inputSchema', () => { + for (const tool of EVAL_TOOL_DEFINITIONS) { + expect(typeof tool.name).toBe('string'); + expect(typeof tool.description).toBe('string'); + expect(tool.description.length).toBeGreaterThan(0); + expect(tool.inputSchema).toHaveProperty('type', 'object'); + expect(tool.inputSchema).toHaveProperty('properties'); + } + }); + + it('all tools require agent_url', () => { + for (const tool of EVAL_TOOL_DEFINITIONS) { + expect(tool.inputSchema.required).toContain('agent_url'); + } + }); + + it('test_rfp_response requires rfp parameter', () => { + const tool = EVAL_TOOL_DEFINITIONS.find((t) => t.name === 'test_rfp_response'); + expect(tool!.inputSchema.required).toContain('rfp'); + }); + + it('test_io_execution requires line_items parameter', () => { + const tool = EVAL_TOOL_DEFINITIONS.find((t) => t.name === 'test_io_execution'); + expect(tool!.inputSchema.required).toContain('line_items'); + }); + + it('evaluate_agent_quality has tracks and platform_type params', () => { + const tool = EVAL_TOOL_DEFINITIONS.find((t) => t.name === 'evaluate_agent_quality'); + expect(tool!.inputSchema.properties).toHaveProperty('tracks'); + expect(tool!.inputSchema.properties).toHaveProperty('platform_type'); + }); +}); + +describe('AGENT_CONTEXT_TOOL_DEFINITIONS', () => { + const EXPECTED = ['save_agent', 'list_saved_agents', 'remove_saved_agent']; + + it('exports exactly the 3 agent context tools', () => { + const names = AGENT_CONTEXT_TOOL_DEFINITIONS.map((t) => t.name); + expect(names).toEqual(expect.arrayContaining(EXPECTED)); + expect(names).toHaveLength(EXPECTED.length); + }); + + it('save_agent requires agent_url', () => { + const tool = AGENT_CONTEXT_TOOL_DEFINITIONS.find((t) => t.name === 'save_agent'); + expect(tool!.inputSchema.required).toContain('agent_url'); + }); + + it('save_agent supports auth_token and protocol params', () => { + const tool = AGENT_CONTEXT_TOOL_DEFINITIONS.find((t) => t.name === 'save_agent'); + expect(tool!.inputSchema.properties).toHaveProperty('auth_token'); + expect(tool!.inputSchema.properties).toHaveProperty('protocol'); + }); + + it('remove_saved_agent requires agent_url', () => { + const tool = AGENT_CONTEXT_TOOL_DEFINITIONS.find((t) => t.name === 'remove_saved_agent'); + expect(tool!.inputSchema.required).toContain('agent_url'); + }); +}); + +describe('SCHEMA_TOOL_DEFINITIONS', () => { + const EXPECTED = ['validate_json', 'get_schema']; + + it('exports exactly the 2 schema tools', () => { + const names = SCHEMA_TOOL_DEFINITIONS.map((t) => t.name); + expect(names).toEqual(expect.arrayContaining(EXPECTED)); + expect(names).toHaveLength(EXPECTED.length); + }); + + it('validate_json requires json parameter', () => { + const tool = SCHEMA_TOOL_DEFINITIONS.find((t) => t.name === 'validate_json'); + expect(tool!.inputSchema.required).toContain('json'); + }); + + it('get_schema requires schema_path parameter', () => { + const tool = SCHEMA_TOOL_DEFINITIONS.find((t) => t.name === 'get_schema'); + expect(tool!.inputSchema.required).toContain('schema_path'); + }); +}); + +describe('PROPERTY_TOOL_DEFINITIONS', () => { + it('exports validate_adagents', () => { + expect(PROPERTY_TOOL_DEFINITIONS).toHaveLength(1); + expect(PROPERTY_TOOL_DEFINITIONS[0].name).toBe('validate_adagents'); + }); + + it('validate_adagents requires domain', () => { + expect(PROPERTY_TOOL_DEFINITIONS[0].inputSchema.required).toContain('domain'); + }); +}); + +describe('ALL_EXPOSED_TOOL_DEFINITIONS', () => { + it('combines all tool groups (4 eval + 3 context + 2 schema + 1 property = 10)', () => { + expect(ALL_EXPOSED_TOOL_DEFINITIONS).toHaveLength(10); + }); + + it('has no duplicate tool names', () => { + const names = ALL_EXPOSED_TOOL_DEFINITIONS.map((t) => t.name); + expect(new Set(names).size).toBe(names.length); + }); + + it('every tool has valid MCP format', () => { + for (const tool of ALL_EXPOSED_TOOL_DEFINITIONS) { + expect(typeof tool.name).toBe('string'); + expect(typeof tool.description).toBe('string'); + expect(tool.inputSchema).toHaveProperty('type', 'object'); + expect(tool.inputSchema).toHaveProperty('properties'); + } + }); +}); + +describe('createMemberToolHandler', () => { + it('returns a function with 2 params (args, authContext)', () => { + const handler = createMemberToolHandler('probe_adcp_agent'); + expect(typeof handler).toBe('function'); + expect(handler.length).toBe(2); + }); + + it('returns isError when called without auth', async () => { + const handler = createMemberToolHandler('probe_adcp_agent'); + const result = await handler({ agent_url: 'https://example.com' }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain('Authentication required'); + }); + + it('returns isError for anonymous auth context', async () => { + const handler = createMemberToolHandler('save_agent'); + const result = await handler( + { agent_url: 'https://example.com' }, + { sub: 'anonymous', isM2M: false, payload: {} }, + ); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain('Authentication required'); + }); +}); + +describe('createStatelessToolHandlers', () => { + it('returns handlers for schema and property tools', () => { + const handlers = createStatelessToolHandlers(); + expect(handlers.has('validate_json')).toBe(true); + expect(handlers.has('get_schema')).toBe(true); + expect(handlers.has('validate_adagents')).toBe(true); + }); + + it('handlers are functions', () => { + const handlers = createStatelessToolHandlers(); + for (const [, handler] of handlers) { + expect(typeof handler).toBe('function'); + } + }); +});