From 66af47d43525945f3b0a5bef0e127d76953eddae Mon Sep 17 00:00:00 2001 From: mantrakp04 Date: Mon, 23 Mar 2026 10:48:24 -0700 Subject: [PATCH 01/51] Enhance documentation tools integration - Added new internal API endpoint for documentation tools, allowing actions such as listing available docs, searching, and fetching specific documentation by ID. - Updated environment configuration to support optional internal secret for enhanced security. - Refactored existing search functionality to utilize the new docs tools API instead of the previous MCP server. - Improved error handling and response parsing for documentation-related requests. - Expanded documentation to clarify the relationship between the new tools and existing API functionalities. This update streamlines the documentation access process and enhances the overall developer experience. --- apps/backend/.env.development | 4 + .../app/api/latest/ai/query/[mode]/route.ts | 2 +- apps/backend/src/lib/ai/tools/docs.ts | 136 +++- claude/CLAUDE-KNOWLEDGE.md | 3 + docs/.env.development | 2 + .../docs/(guides)/others/mcp-setup.mdx | 11 +- .../src/app/api/internal/[transport]/route.ts | 612 ++---------------- .../[transport]/setup-instructions.md | 3 +- docs/src/app/api/internal/docs-tools/route.ts | 55 ++ docs/src/app/api/search/route.ts | 94 +-- docs/src/app/mcp-browser/page.tsx | 52 +- docs/src/lib/docs-tools-operations.ts | 491 ++++++++++++++ packages/private | 2 +- 13 files changed, 767 insertions(+), 700 deletions(-) create mode 100644 docs/src/app/api/internal/docs-tools/route.ts create mode 100644 docs/src/lib/docs-tools-operations.ts diff --git a/apps/backend/.env.development b/apps/backend/.env.development index 02fcc18611..dbaf4959d5 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -76,6 +76,10 @@ STACK_OPENAI_API_KEY=mock_openai_api_key STACK_STRIPE_SECRET_KEY=sk_test_mockstripekey STACK_STRIPE_WEBHOOK_SECRET=mock_stripe_webhook_secret STACK_OPENROUTER_API_KEY=FORWARD_TO_PRODUCTION +# Optional: override docs origin for the `docs` AI tool bundle (defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod) +# STACK_DOCS_INTERNAL_BASE_URL=http://localhost:8104 +# Optional: shared secret; when set, backend sends it and docs `/api/internal/docs-tools` requires it +# STACK_INTERNAL_DOCS_TOOLS_SECRET= # Email monitor configuration for tests STACK_EMAIL_MONITOR_VERIFICATION_CALLBACK_URL=http://localhost:8101/handler/email-verification STACK_EMAIL_MONITOR_PROJECT_ID=internal diff --git a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts index aded240d53..ea75c58423 100644 --- a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts +++ b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts @@ -129,7 +129,7 @@ export const POST = createSmartRouteHandler({ return { statusCode: 200, bodyType: "json" as const, - body: { content: contentBlocks }, + body: { content: contentBlocks, finalText: result.text }, }; } }, diff --git a/apps/backend/src/lib/ai/tools/docs.ts b/apps/backend/src/lib/ai/tools/docs.ts index 61d305c0a8..37aa235d1b 100644 --- a/apps/backend/src/lib/ai/tools/docs.ts +++ b/apps/backend/src/lib/ai/tools/docs.ts @@ -1,21 +1,131 @@ -import { createMCPClient } from "@ai-sdk/mcp"; -import { getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { tool } from "ai"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { z } from "zod"; + +type DocsToolHttpResult = { + content?: Array<{ type: string, text?: string }>, + isError?: boolean, +}; + +function getDocsToolsBaseUrl(): string { + const fromEnv = getEnvVariable("STACK_DOCS_INTERNAL_BASE_URL", ""); + if (fromEnv !== "") { + return fromEnv.replace(/\/$/, ""); + } + if (getNodeEnvironment() === "development") { + const portPrefix = getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81"); + return `http://localhost:${portPrefix}04`; + } + return "https://mcp.stack-auth.com"; +} + +async function postDocsToolAction(action: Record): Promise { + const base = getDocsToolsBaseUrl(); + const secret = getEnvVariable("STACK_INTERNAL_DOCS_TOOLS_SECRET", ""); + const headers: Record = { + "Content-Type": "application/json", + }; + if (secret !== "") { + headers["x-stack-internal-docs-tools-secret"] = secret; + } + + const res = await fetch(`${base}/api/internal/docs-tools`, { + method: "POST", + headers, + body: JSON.stringify(action), + }); + + if (!res.ok) { + const errBody = await res.text(); + return `Stack Auth docs tools error (${res.status}): ${errBody}`; + } + + const data = (await res.json()) as DocsToolHttpResult; + const text = data.content + ?.filter((c): c is { type: "text", text: string } => c.type === "text" && typeof c.text === "string") + .map((c) => c.text) + .join("\n") ?? ""; + + if (data.isError === true) { + return text || "Unknown docs tool error"; + } + + return text; +} /** - * Creates an MCP client connected to the Stack Auth documentation server. + * Documentation tools backed by the docs app's `/api/internal/docs-tools` endpoint. * - * In development: connects to local docs server at http://localhost:8104 - * In production: connects to production docs server at https://mcp.stack-auth.com + * The public MCP server at the same docs origin exposes only `ask_stack_auth`, which proxies to + * `/api/latest/ai/query/generate`; these tools avoid MCP recursion by calling the HTTP API directly. */ export async function createDocsTools() { - const mcpUrl = - getNodeEnvironment() === "development" - ? new URL("/api/internal/mcp", "http://localhost:8104") - : new URL("/api/internal/mcp", "https://mcp.stack-auth.com"); + return { + list_available_docs: tool({ + description: + "Use this tool to learn about what Stack Auth is, available documentation, and see if you can use it for what you're working on. It returns a list of all available Stack Auth Documentation pages.", + inputSchema: z.object({}), + execute: async () => { + return await postDocsToolAction({ action: "list_available_docs" }); + }, + }), - const stackAuthMcp = await createMCPClient({ - transport: { type: "http", url: mcpUrl.toString() }, - }); + search_docs: tool({ + description: + "Search through all Stack Auth documentation including API docs, guides, and examples. Returns ranked results with snippets and relevance scores.", + inputSchema: z.object({ + search_query: z.string().describe("The search query to find relevant documentation"), + result_limit: z.number().optional().describe("Maximum number of results to return (default: 50)"), + }), + execute: async ({ search_query, result_limit = 50 }) => { + return await postDocsToolAction({ + action: "search_docs", + search_query, + result_limit, + }); + }, + }), + + get_docs_by_id: tool({ + description: + "Use this tool to retrieve a specific Stack Auth Documentation page by its ID. It gives you the full content of the page so you can know exactly how to use specific Stack Auth APIs. Whenever using Stack Auth, you should always check the documentation first to have the most up-to-date information. When you write code using Stack Auth documentation you should reference the content you used in your comments.", + inputSchema: z.object({ + id: z.string(), + }), + execute: async ({ id }) => { + return await postDocsToolAction({ action: "get_docs_by_id", id }); + }, + }), + + get_stack_auth_setup_instructions: tool({ + description: + "Use this tool when the user wants to set up authentication in a new project. It provides step-by-step instructions for installing and configuring Stack Auth authentication.", + inputSchema: z.object({}), + execute: async () => { + return await postDocsToolAction({ action: "get_stack_auth_setup_instructions" }); + }, + }), + + search: tool({ + description: + "Search for Stack Auth documentation pages.\n\nUse this tool to find documentation pages that contain a specific keyword or phrase.", + inputSchema: z.object({ + query: z.string(), + }), + execute: async ({ query }) => { + return await postDocsToolAction({ action: "search", query }); + }, + }), - return await stackAuthMcp.tools(); + fetch: tool({ + description: + "Fetch a particular Stack Auth Documentation page by its ID.\n\nThis tool is identical to `get_docs_by_id`.", + inputSchema: z.object({ + id: z.string(), + }), + execute: async ({ id }) => { + return await postDocsToolAction({ action: "fetch", id }); + }, + }), + }; } diff --git a/claude/CLAUDE-KNOWLEDGE.md b/claude/CLAUDE-KNOWLEDGE.md index 4d5b54ba9e..03c2cb3e36 100644 --- a/claude/CLAUDE-KNOWLEDGE.md +++ b/claude/CLAUDE-KNOWLEDGE.md @@ -99,3 +99,6 @@ A: Update affected inline snapshots in `apps/e2e/tests/backend/endpoints/api/v1/ Q: How should `createOrUpdateProjectWithLegacyConfig` handle `onboardingStatus` for forward-compat checks? A: Only write `onboardingStatus` when the `Project.onboardingStatus` column exists (for example by checking `information_schema.columns` in-transaction) so current code can still run against older schemas where that column is absent. + +Q: How does the Stack Auth docs MCP relate to the ask-chat API and doc tools? +A: The public MCP (`/api/internal/mcp` on the docs site) exposes only `ask_stack_auth`, which POSTs to `/api/latest/ai/query/generate` with `tools: ["docs"]` and `systemPrompt: "docs-ask-ai"`. The backend no longer loads doc tools via MCP; `createDocsTools()` calls the docs app `POST /api/internal/docs-tools` with typed actions (same behavior as before). Optional `STACK_INTERNAL_DOCS_TOOLS_SECRET` gates the internal route; `STACK_DOCS_INTERNAL_BASE_URL` overrides the docs origin for the backend. diff --git a/docs/.env.development b/docs/.env.development index 7f67d50ea3..cd0ca772f6 100644 --- a/docs/.env.development +++ b/docs/.env.development @@ -5,3 +5,5 @@ NEXT_PUBLIC_STACK_PROJECT_ID=internal NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=this-publishable-client-key-is-for-local-development-only STACK_SECRET_SERVER_KEY=this-secret-server-key-is-for-local-development-only STACK_OPENROUTER_API_KEY=your-open-router-api-key-for-ai-enabled-chat +# Optional: require this header value for POST /api/internal/docs-tools (must match backend STACK_INTERNAL_DOCS_TOOLS_SECRET) +# STACK_INTERNAL_DOCS_TOOLS_SECRET= diff --git a/docs/content/docs/(guides)/others/mcp-setup.mdx b/docs/content/docs/(guides)/others/mcp-setup.mdx index 2521595b70..047818527f 100644 --- a/docs/content/docs/(guides)/others/mcp-setup.mdx +++ b/docs/content/docs/(guides)/others/mcp-setup.mdx @@ -10,6 +10,8 @@ title: MCP Setup Set up Stack Auth's Model Context Protocol (MCP) server to get intelligent code assistance in your development environment. +The MCP server exposes a single tool, **`ask_stack_auth`**, which sends your question to the same Stack Auth documentation AI used on [docs.stack-auth.com](https://docs.stack-auth.com). + Cursor @@ -231,10 +233,9 @@ by [![Hypr MCP](https://hyprmcp.com/hyprmcp_20px.svg)](https://hyprmcp.com/)`}]+)>/); - if (componentMatch) { - const props = componentMatch[1]; - const documentMatch = props.match(/document=\{"([^"]+)"\}/); - const operationsMatch = props.match(/operations=\{(\[[^\]]+\])\}/); - - if (documentMatch && operationsMatch) { - const specFile = documentMatch[1]; - const operations = operationsMatch[1]; - - try { - const specPath = specFile; - const specContent = await readFile(specPath, "utf-8"); - const spec = JSON.parse(specContent); - const parsedOps = JSON.parse(operations); - let apiDetails = ''; - - for (const op of parsedOps) { - const { path: opPath, method } = op; - const pathSpec = spec.paths?.[opPath]; - const methodSpec = pathSpec?.[method.toLowerCase()]; - - if (methodSpec) { - // Add human-readable summary first - const fullUrl = methodSpec['x-full-url'] || `https://api.stack-auth.com/api/v1${opPath}`; - - apiDetails += `\n## ${method.toUpperCase()} ${opPath}\n`; - apiDetails += `**Full URL:** ${fullUrl}\n`; - apiDetails += `**Summary:** ${methodSpec.summary || 'No summary available'}\n\n`; - - // Then include the complete OpenAPI spec with all examples and schemas - const endpointJson = { - [opPath]: { - [method.toLowerCase()]: methodSpec - } - }; - apiDetails += "**Complete API Specification:**\n```json\n"; - apiDetails += JSON.stringify(endpointJson, null, 2); - apiDetails += "\n```\n\n---\n"; - } - } - - const resultText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\n\n${apiDetails}`; - - return { - content: [ - { - type: "text" as const, - text: resultText, - }, - ], - }; - } catch (specError) { - const errorText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\nError reading OpenAPI spec: ${specError instanceof Error ? specError.message : "Unknown error"}`; - - return { - content: [ - { - type: "text" as const, - text: errorText, - }, - ], - }; - } - } - } - - // If no component match or missing props, return regular content - const fallbackText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\nContent:\n${content}`; - - return { - content: [ - { - type: "text" as const, - text: fallbackText, - }, - ], - }; -} - -// Get pages from both main docs and API docs -const pages = source.getPages(); -const apiPages = apiSource.getPages(); - -// Filter out admin API pages from the MCP server -const filteredApiPages = apiPages.filter((page) => { - // Exclude admin API pages - they should not be accessible via MCP - return !page.url.startsWith('/api/admin/'); -}); - -const allPages = [...pages, ...filteredApiPages]; - -// Helper to extract actual API endpoint from page frontmatter -function getApiEndpointFromPage(page: typeof allPages[0]): string | null { - if (!page.url.startsWith('/api/') || page.url.startsWith('/api/webhooks/')) { - return null; - } - - // Check if the page data has _openapi metadata - const pageData = page.data as { _openapi?: { method?: string, route?: string } }; - - if (pageData._openapi && pageData._openapi.method && pageData._openapi.route) { - const endpoint = `${pageData._openapi.method.toUpperCase()} ${pageData._openapi.route}`; - return endpoint; - } - - return null; -} - -const pageSummaries = allPages - .filter((v) => { - return !(v.slugs[0] == "API-Reference"); - }) - .map((page) => - ` -Title: ${page.data.title} -Description: ${page.data.description} -ID: ${page.url} -`.trim() - ) - .join("\n"); - -async function getDocsById({ id }: { id: string }): Promise { - nodeClient?.capture({ - event: "get_docs_by_id", - properties: { id }, - distinctId: "mcp-handler", - }); - const page = allPages.find((page) => page.url === id); - if (!page) { - return { content: [{ type: "text", text: "Page not found." }] }; - } - // Check if this is an API page and handle OpenAPI spec extraction - const isApiPage = page.url.startsWith("/api/"); - - // Try primary path first, then fallback to docs/ prefix or api/ prefix - const filePath = `content/${page.file.path}`; - try { - const content = await readFile(filePath, "utf-8"); - - if (isApiPage && content.includes(" { server.tool( - "list_available_docs", - "Use this tool to learn about what Stack Auth is, available documentation, and see if you can use it for what you're working on. It returns a list of all available Stack Auth Documentation pages.", - {}, - async ({}) => { - nodeClient?.capture({ - event: "list_available_docs", - properties: {}, - distinctId: "mcp-handler", - }); - return { - content: [{ type: "text", text: pageSummaries }], - }; - }, - ); - server.tool( - "search_docs", - "Search through all Stack Auth documentation including API docs, guides, and examples. Returns ranked results with snippets and relevance scores.", + "ask_stack_auth", + "Ask the Stack Auth documentation assistant. Use this for any question about Stack Auth: setup, APIs, SDK usage, configuration, or troubleshooting. The assistant searches official documentation and answers with citations. Always set `reason` to a short explanation of why you are calling this tool (for product analytics and debugging).", { - search_query: z.string().describe("The search query to find relevant documentation"), - result_limit: z.number().optional().describe("Maximum number of results to return (default: 50)") + question: z.string().describe("The full question to ask about Stack Auth."), + reason: z + .string() + .min(1) + .describe( + "Why the agent invoked this tool (e.g. user asked about OAuth setup, need Stack Auth API headers). Used for analytics, not sent to the model.", + ), }, - async ({ search_query, result_limit = 50 }) => { + async ({ question, reason }) => { nodeClient?.capture({ - event: "search_docs", - properties: { search_query, result_limit }, + event: "ask_stack_auth_mcp", + properties: { question, reason }, distinctId: "mcp-handler", }); - type SearchResult = { - title: string, - description: string, - url: string, - score: number, - snippet: string, - type: 'api' | 'docs', - apiEndpoint?: string | null, - }; - - const results: SearchResult[] = []; - const queryLower = search_query.toLowerCase().trim(); - const queryWords = queryLower.split(/\s+/).filter(w => w.length > 0); - - // Search through all pages - for (const page of allPages) { - // Skip admin API endpoints - if (page.url.startsWith('/api/admin/')) { - continue; - } - - let score = 0; - const title = page.data.title || ''; - const description = page.data.description || ''; - const titleLower = title.toLowerCase(); - const descriptionLower = description.toLowerCase(); - - // Exact phrase match in title (highest priority) - if (titleLower.includes(queryLower)) { - if (titleLower === queryLower) { - score += 100; // Exact match - } else if (titleLower.startsWith(queryLower)) { - score += 80; // Starts with - } else { - score += 60; // Contains - } - } - - // Individual word matching in title - for (const word of queryWords) { - if (titleLower.includes(word)) { - score += 30; // Each word match - } - } - - // Exact phrase match in description - if (descriptionLower.includes(queryLower)) { - score += 40; - } - - // Individual word matching in description - for (const word of queryWords) { - if (descriptionLower.includes(word)) { - score += 15; // Each word match - } - } - // TOC/heading matching - for (const tocItem of page.data.toc) { - if (typeof tocItem.title === 'string') { - const tocTitleLower = tocItem.title.toLowerCase(); - // Exact phrase match - if (tocTitleLower.includes(queryLower)) { - score += 30; - } - // Individual word matching - for (const word of queryWords) { - if (tocTitleLower.includes(word)) { - score += 10; - } - } - } - } - - // Content matching (try to read the actual file) - try { - // Fix the file path - fumadocs page.file.path doesn't include 'api/' prefix - let filePath = `content/${page.file.path}`; - // If it's an API page and the path doesn't start with api/, add it - if (page.url.startsWith('/api/') && !page.file.path.startsWith('api/')) { - filePath = `content/api/${page.file.path}`; - } - const content = await readFile(filePath, "utf-8"); - const textContent = content - .replace(/^---[\s\S]*?---/, '') // Remove frontmatter - .replace(/<[^>]*>/g, ' ') // Remove JSX tags - .replace(/\{[^}]*\}/g, ' ') // Remove JSX expressions - .replace(/```[a-zA-Z]*\n/g, ' ') // Remove code block markers - .replace(/```/g, ' ') - .replace(/`([^`]*)`/g, '$1') // Remove inline code backticks - .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') // Extract link text - .replace(/[#*_~]/g, '') // Remove markdown formatting - .replace(/\s+/g, ' ') - .trim(); - - const textContentLower = textContent.toLowerCase(); - - // Exact phrase match in content - let hasContentMatch = false; - if (textContentLower.includes(queryLower)) { - score += 20; - hasContentMatch = true; - } - - // Individual word matching in content - for (const word of queryWords) { - if (textContentLower.includes(word)) { - score += 5; // Each word match - hasContentMatch = true; - } - } - - if (hasContentMatch && queryWords.length > 0) { - // Find snippet around the first query word match - const firstWord = queryWords[0]; - const matchIndex = textContentLower.indexOf(firstWord); - const start = Math.max(0, matchIndex - 50); - const end = Math.min(textContent.length, matchIndex + 100); - const snippet = textContent.slice(start, end); - - // For API pages, try to extract the actual endpoint - const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; - - results.push({ - title, - description, - url: page.url, - score, - snippet: `...${snippet}...`, - type: page.url.startsWith('/api/') ? 'api' : 'docs', - apiEndpoint - }); - } else if (score > 0) { - // Add without snippet if title/description matched - const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; - - results.push({ - title, - description, - url: page.url, - score, - snippet: description || title, - type: page.url.startsWith('/api/') ? 'api' : 'docs', - apiEndpoint - }); - } - } catch (error) { - // If file reading fails but we have title/description matches - if (score > 0) { - const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; - - results.push({ - title, - description, - url: page.url, - score, - snippet: description || title, - type: page.url.startsWith('/api/') ? 'api' : 'docs', - apiEndpoint - }); - } - } + const apiBase = process.env.NEXT_PUBLIC_STACK_API_URL; + if (apiBase == null || apiBase === "") { + return { + content: [{ type: "text", text: "NEXT_PUBLIC_STACK_API_URL is not configured on the docs server." }], + isError: true, + }; } - // Sort by score (highest first) and limit results - const sortedResults = results - .sort((a, b) => b.score - a.score) - .slice(0, result_limit); - - const searchResultText = sortedResults.length > 0 - ? sortedResults.map(result => { - let text = `Title: ${result.title}\nDescription: ${result.description}\n`; - - if (result.apiEndpoint) { - text += `API Endpoint: ${result.apiEndpoint}\n`; - } - - text += `Documentation URL: ${result.url}\nType: ${result.type}\nScore: ${result.score}\nSnippet: ${result.snippet}\n`; - - return text; - }).join('\n---\n') - : `No results found for "${search_query}"`; - - return { - content: [{ type: "text", text: searchResultText }], - }; - } - ); - server.tool( - "get_docs_by_id", - "Use this tool to retrieve a specific Stack Auth Documentation page by its ID. It gives you the full content of the page so you can know exactly how to use specific Stack Auth APIs. Whenever using Stack Auth, you should always check the documentation first to have the most up-to-date information. When you write code using Stack Auth documentation you should reference the content you used in your comments.", - { id: z.string() }, - getDocsById, - ); - server.tool( - "get_stack_auth_setup_instructions", - "Use this tool when the user wants to set up authentication in a new project. It provides step-by-step instructions for installing and configuring Stack Auth authentication.", - {}, - async () => { - nodeClient?.capture({ - event: "get_stack_auth_setup_instructions", - properties: {}, - distinctId: "mcp-handler", + const url = `${apiBase.replace(/\/$/, "")}/api/latest/ai/query/generate`; + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + quality: "smart", + speed: "fast", + tools: ["docs"], + systemPrompt: "docs-ask-ai", + messages: [{ role: "user", content: question }], + }), }); - try { - const instructionsFromCurrentFile = "content/setup-instructions.md"; - const instructions = await readFile(instructionsFromCurrentFile, "utf-8"); - - return { - content: [ - { - type: "text" as const, - text: instructions, - }, - ], - }; - } catch (error) { + if (!res.ok) { + const errText = await res.text(); return { - content: [ - { - type: "text" as const, - text: `Error reading setup instructions: ${error instanceof Error ? error.message : "Unknown error"}`, - }, - ], + content: [{ type: "text", text: `Stack Auth AI error (${res.status}): ${errText}` }], isError: true, }; } - }, - ); - // Search tool for ChatGPT deep research - // Reference: https://platform.openai.com/docs/mcp#search-tool - server.tool( - "search", - "Search for Stack Auth documentation pages.\n\nUse this tool to find documentation pages that contain a specific keyword or phrase.", - { query: z.string() }, - async ({ query }) => { - nodeClient?.capture({ - event: "search", - properties: { query }, - distinctId: "mcp-handler", - }); + const body = (await res.json()) as { + finalText?: string, + content?: Array<{ type: string, text?: string }>, + }; + + const text = + body.finalText ?? + body.content + ?.filter((c): c is { type: "text", text: string } => c.type === "text" && typeof c.text === "string") + .map((c) => c.text) + .join("\n\n") ?? + ""; - const q = query.toLowerCase(); - const results = allPages - .filter( - (page) => - page.data.title.toLowerCase().includes(q) || - page.data.description?.toLowerCase().includes(q), - ) - .map((page) => ({ - id: page.url, - title: page.data.title, - url: page.url, - })); return { - content: [ - { - type: "text", - text: JSON.stringify({ results }), - }, - ], + content: [{ type: "text", text: text.length > 0 ? text : "(empty response)" }], }; }, ); - - // Fetch tool for ChatGPT deep research - // Reference: https://platform.openai.com/docs/mcp#fetch-tool - server.tool( - "fetch", - "Fetch a particular Stack Auth Documentation page by its ID.\n\nThis tool is identical to `get_docs_by_id`.", - { id: z.string() }, - getDocsById, - ); }, { capabilities: { tools: { - listAvailableDocs: { - description: - "Use this tool to learn about what Stack Auth is, available documentation, and see if you can use it for what you're working on. It returns a list of all available Stack Auth Documentation pages.", - }, - getDocById: { + askStackAuth: { description: - "Use this tool to retrieve a specific Stack Auth Documentation page by its ID. It gives you the full content of the page so you can know exactly how to use specific Stack Auth APIs. Whenever using Stack Auth, you should always check the documentation first to have the most up-to-date information. When you write code using Stack Auth documentation you should reference the content you used in your comments.", + "Ask the Stack Auth documentation assistant any question about Stack Auth (setup, APIs, SDKs, configuration, troubleshooting).", parameters: { type: "object", properties: { - id: { + question: { type: "string", - description: "The ID of the documentation page to retrieve.", + description: "The full question to ask about Stack Auth.", }, - }, - required: ["id"], - }, - }, - getStackAuthSetupInstructions: { - description: - "Use this tool when the user wants to set up Stack Auth in a new project. It provides step-by-step instructions for installing and configuring Stack Auth authentication, including environment setup, file scaffolding, and verification steps.", - parameters: { - type: "object", - properties: {}, - required: [], - }, - }, - search: { - description: - "Search for Stack Auth documentation pages.\n\nUse this tool to find documentation pages that contain a specific keyword or phrase.", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "The search query to use.", - }, - }, - required: ["query"], - }, - }, - fetch: { - description: - "Fetch a particular Stack Auth Documentation page by its ID.\n\nThis tool is identical to `get_docs_by_id`.", - parameters: { - type: "object", - properties: { - id: { + reason: { type: "string", - description: "The ID of the documentation page to retrieve.", + description: + "Why the agent invoked this tool (for analytics and debugging). Not sent to the documentation model.", }, }, - required: ["id"], + required: ["question", "reason"], }, }, }, @@ -605,7 +103,7 @@ const handler = createMcpHandler( { basePath: "/api/internal", verboseLogs: true, - maxDuration: 60, + maxDuration: 120, } ); diff --git a/docs/src/app/api/internal/[transport]/setup-instructions.md b/docs/src/app/api/internal/[transport]/setup-instructions.md index a86b552b66..5c2a11d8ee 100644 --- a/docs/src/app/api/internal/[transport]/setup-instructions.md +++ b/docs/src/app/api/internal/[transport]/setup-instructions.md @@ -20,8 +20,7 @@ Before proceeding, you MUST identify the project framework: **IMPORTANT**: Only proceed with the installation if you can clearly identify the project as either Next.js or React. ### 1) Run the Stack Auth initializer -- Use the `stack-auth` MCP server. -- Call the tool or run the command: +- Use the `stack-auth` MCP server (`ask_stack_auth` tool), or run the command: - **For Next.js projects**: ```bash npx @stackframe/stack-cli@latest init diff --git a/docs/src/app/api/internal/docs-tools/route.ts b/docs/src/app/api/internal/docs-tools/route.ts new file mode 100644 index 0000000000..60dca3bd43 --- /dev/null +++ b/docs/src/app/api/internal/docs-tools/route.ts @@ -0,0 +1,55 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { type DocsToolAction, executeDocsToolAction } from "@/lib/docs-tools-operations"; + +const bodySchema: z.ZodType = z.discriminatedUnion("action", [ + z.object({ action: z.literal("list_available_docs") }), + z.object({ + action: z.literal("search_docs"), + search_query: z.string(), + result_limit: z.number().optional(), + }), + z.object({ action: z.literal("get_docs_by_id"), id: z.string() }), + z.object({ action: z.literal("get_stack_auth_setup_instructions") }), + z.object({ action: z.literal("search"), query: z.string() }), + z.object({ action: z.literal("fetch"), id: z.string() }), +]); + +function validateInternalSecret(req: NextRequest): boolean { + const secret = process.env.STACK_INTERNAL_DOCS_TOOLS_SECRET; + if (secret == null || secret === "") { + return true; + } + return req.headers.get("x-stack-internal-docs-tools-secret") === secret; +} + +export async function POST(req: NextRequest) { + if (!validateInternalSecret(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let json: unknown; + try { + json = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON" }, { status: 400 }); + } + + const parsed = bodySchema.safeParse(json); + if (!parsed.success) { + return NextResponse.json( + { error: "Invalid body", details: parsed.error.flatten() }, + { status: 400 }, + ); + } + + const result = await executeDocsToolAction(parsed.data); + return NextResponse.json(result); +} + +export async function GET() { + return NextResponse.json( + { error: "Use POST with a DocsToolAction body" }, + { status: 405 }, + ); +} diff --git a/docs/src/app/api/search/route.ts b/docs/src/app/api/search/route.ts index 9159c39c82..92bcb115e0 100644 --- a/docs/src/app/api/search/route.ts +++ b/docs/src/app/api/search/route.ts @@ -8,92 +8,30 @@ type SearchResult = { title?: string, }; -// Helper function to call MCP server -async function callMcpServer(search_query: string): Promise { +// Helper: same search implementation as MCP / backend docs tools (internal HTTP API) +async function callDocsToolsSearch(search_query: string, requestOrigin: string): Promise { try { - // Use localhost during development, production URL otherwise - const mcpUrl = process.env.NODE_ENV === 'development' - ? 'http://localhost:8104/api/internal/mcp' - : 'https://mcp.stack-auth.com/api/internal/mcp'; - - console.log(`Calling MCP server at: ${mcpUrl}`); - - const response = await fetch(mcpUrl, { + const response = await fetch(`${requestOrigin}/api/internal/docs-tools`, { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Accept': 'application/json, text/event-stream', }, body: JSON.stringify({ - jsonrpc: '2.0', - method: 'tools/call', - params: { - name: 'search_docs', - arguments: { search_query, result_limit: 20 }, - }, - id: Date.now(), + action: 'search_docs', + search_query, + result_limit: 20, }), }); if (!response.ok) { const errorText = await response.text(); - console.error(`MCP server error (${response.status}):`, errorText); - throw new Error(`MCP server error: ${response.status} - ${errorText.substring(0, 200)}`); - } - - // Parse Server-Sent Events format response - // Read the stream until we get the data event (don't wait for connection to close) - if (!response.body) { - throw new Error('No response body'); - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - let jsonData = null; - - try { - while (true) { - const { done, value } = await reader.read(); - - if (value) { - buffer += decoder.decode(value, { stream: true }); - - // Look for complete data: lines in the buffer - const lines = buffer.split('\n'); - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - jsonData = JSON.parse(line.substring(6)); - // Found our data, we can stop reading - await reader.cancel(); - break; - } catch (e) { - // Continue looking for valid JSON data - } - } - } - - if (jsonData) break; - } - - if (done) break; - } - } finally { - reader.releaseLock(); + console.error(`docs-tools error (${response.status}):`, errorText); + throw new Error(`docs-tools error: ${response.status} - ${errorText.substring(0, 200)}`); } - if (!jsonData) { - throw new Error('Invalid MCP response format'); - } - - if (jsonData.error) { - throw new Error(jsonData.error.message || 'MCP search failed'); - } + const jsonData = (await response.json()) as { content?: Array<{ type: string, text?: string }> }; - // Parse the search results from the text response - const searchResultText = jsonData.result?.content?.[0]?.text || ''; + const searchResultText = jsonData.content?.[0]?.text || ''; if (searchResultText.includes('No results found')) { return []; } @@ -139,7 +77,7 @@ async function callMcpServer(search_query: string): Promise { return results; } catch (error) { - console.error('MCP server call failed:', error); + console.error('docs-tools search failed:', error); // Fallback to empty results return []; } @@ -167,20 +105,20 @@ export async function GET(request: NextRequest) { } try { - // Call MCP server for search results - const results = await callMcpServer(search_query); + const origin = new URL(request.url).origin; + const results = await callDocsToolsSearch(search_query, origin); - console.log(`Found ${results.length} search results from MCP server for "${search_query}"`); + console.log(`Found ${results.length} search results from docs-tools for "${search_query}"`); // Filter out admin API endpoints as an additional safety measure const filteredResults = results.filter(result => !result.url.startsWith('/api/admin')); - // Sort by platform priority since MCP server already handles relevance + // Sort by platform priority since docs-tools already handles relevance const sortedResults = filteredResults.sort((a, b) => { return getPlatformPriority(b.url) - getPlatformPriority(a.url); }); - console.log(`\n=== MCP SEARCH RESULTS FOR "${search_query}" ===`); + console.log(`\n=== DOCS SEARCH RESULTS FOR "${search_query}" ===`); sortedResults.slice(0, 10).forEach((result, i) => { const priority = getPlatformPriority(result.url); console.log(`${i + 1}. "${result.content}" (${result.type}) - Priority: ${priority} - URL: ${result.url}`); diff --git a/docs/src/app/mcp-browser/page.tsx b/docs/src/app/mcp-browser/page.tsx index 474205f7ab..f1364da3e8 100644 --- a/docs/src/app/mcp-browser/page.tsx +++ b/docs/src/app/mcp-browser/page.tsx @@ -33,56 +33,21 @@ export default function McpBrowserPage() { const [docLoading, setDocLoading] = useState(false); const [error, setError] = useState(null); - // Function to call MCP tools - const callMcpTool = async (toolName: string, args: Record = {}) => { - const response = await fetch('/api/internal/mcp', { + /** Same doc operations as the MCP-exposed `ask_stack_auth` backend path; list/get use the internal JSON API. */ + const callDocsTool = async (body: Record) => { + const response = await fetch('/api/internal/docs-tools', { method: 'POST', headers: { 'Content-Type': 'application/json', - 'Accept': 'application/json, text/event-stream', }, - body: JSON.stringify({ - jsonrpc: '2.0', - method: 'tools/call', - params: { - name: toolName, - arguments: args, - }, - id: Date.now(), - }), + body: JSON.stringify(body), }); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } - // Parse Server-Sent Events format response - const text = await response.text(); - - // Look for the data line in the SSE response - const lines = text.split('\n'); - let jsonData = null; - - for (const line of lines) { - if (line.startsWith('data: ')) { - try { - jsonData = JSON.parse(line.substring(6)); - break; - } catch (e) { - // Continue looking for valid JSON data - } - } - } - - if (!jsonData) { - throw new Error('Invalid MCP response format'); - } - - if (jsonData.error) { - throw new Error(jsonData.error.message || 'MCP tool call failed'); - } - - return jsonData.result; + return await response.json() as { content: Array<{ type: string, text?: string }> }; }; // Parse the doc summaries from the text response @@ -119,7 +84,7 @@ export default function McpBrowserPage() { const loadDocs = async () => { try { setLoading(true); - const result = await callMcpTool('list_available_docs'); + const result = await callDocsTool({ action: 'list_available_docs' }); const textContent = result.content[0]?.text || ''; const parsedDocs = parseDocSummaries(textContent); setDocs(parsedDocs); @@ -140,7 +105,7 @@ export default function McpBrowserPage() { const loadDoc = async (docId: string) => { try { setDocLoading(true); - const result = await callMcpTool('get_docs_by_id', { id: docId }); + const result = await callDocsTool({ action: 'get_docs_by_id', id: docId }); const textContent = result.content[0]?.text || ''; // Parse the response which includes title, description, and content @@ -235,7 +200,8 @@ export default function McpBrowserPage() {

- Browse Stack Auth documentation through the Model Context Protocol server. + Browse Stack Auth documentation using the same sources as the public MCP server (the MCP surface exposes only the{' '} + ask_stack_auth tool; this page uses the internal docs API for listing and viewing pages). Found {docs.length} documentation pages.

diff --git a/docs/src/lib/docs-tools-operations.ts b/docs/src/lib/docs-tools-operations.ts new file mode 100644 index 0000000000..601f865eaf --- /dev/null +++ b/docs/src/lib/docs-tools-operations.ts @@ -0,0 +1,491 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { PostHog } from "posthog-node"; +import { apiSource, source } from "../../lib/source"; + +const nodeClient = process.env.NEXT_PUBLIC_POSTHOG_KEY + ? new PostHog(process.env.NEXT_PUBLIC_POSTHOG_KEY) + : null; + +async function extractOpenApiDetails( + content: string, + page: { data: { title: string, description?: string } }, +): Promise { + const componentMatch = content.match(/]+)>/); + if (componentMatch) { + const props = componentMatch[1]; + const documentMatch = props.match(/document=\{"([^"]+)"\}/); + const operationsMatch = props.match(/operations=\{(\[[^\]]+\])\}/); + + if (documentMatch && operationsMatch) { + const specFile = documentMatch[1]; + const operations = operationsMatch[1]; + + try { + const specPath = specFile; + const specContent = await readFile(specPath, "utf-8"); + const spec = JSON.parse(specContent); + const parsedOps = JSON.parse(operations); + let apiDetails = ''; + + for (const op of parsedOps) { + const { path: opPath, method } = op; + const pathSpec = spec.paths?.[opPath]; + const methodSpec = pathSpec?.[method.toLowerCase()]; + + if (methodSpec) { + const fullUrl = methodSpec['x-full-url'] || `https://api.stack-auth.com/api/v1${opPath}`; + + apiDetails += `\n## ${method.toUpperCase()} ${opPath}\n`; + apiDetails += `**Full URL:** ${fullUrl}\n`; + apiDetails += `**Summary:** ${methodSpec.summary || 'No summary available'}\n\n`; + + const endpointJson = { + [opPath]: { + [method.toLowerCase()]: methodSpec + } + }; + apiDetails += "**Complete API Specification:**\n```json\n"; + apiDetails += JSON.stringify(endpointJson, null, 2); + apiDetails += "\n```\n\n---\n"; + } + } + + const resultText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\n\n${apiDetails}`; + + return { + content: [ + { + type: "text" as const, + text: resultText, + }, + ], + }; + } catch (specError) { + const errorText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\nError reading OpenAPI spec: ${specError instanceof Error ? specError.message : "Unknown error"}`; + + return { + content: [ + { + type: "text" as const, + text: errorText, + }, + ], + }; + } + } + } + + const fallbackText = `Title: ${page.data.title}\nDescription: ${page.data.description || ''}\nContent:\n${content}`; + + return { + content: [ + { + type: "text" as const, + text: fallbackText, + }, + ], + }; +} + +const pages = source.getPages(); +const apiPages = apiSource.getPages(); + +const filteredApiPages = apiPages.filter((page) => { + return !page.url.startsWith('/api/admin/'); +}); + +const allPages = [...pages, ...filteredApiPages]; + +function getApiEndpointFromPage(page: typeof allPages[0]): string | null { + if (!page.url.startsWith('/api/') || page.url.startsWith('/api/webhooks/')) { + return null; + } + + const pageData = page.data as { _openapi?: { method?: string, route?: string } }; + + if (pageData._openapi && pageData._openapi.method && pageData._openapi.route) { + const endpoint = `${pageData._openapi.method.toUpperCase()} ${pageData._openapi.route}`; + return endpoint; + } + + return null; +} + +const pageSummaries = allPages + .filter((v) => { + return !(v.slugs[0] == "API-Reference"); + }) + .map((page) => + ` +Title: ${page.data.title} +Description: ${page.data.description} +ID: ${page.url} +`.trim() + ) + .join("\n"); + +async function getDocsByIdImpl({ id }: { id: string }): Promise { + nodeClient?.capture({ + event: "get_docs_by_id", + properties: { id }, + distinctId: "mcp-handler", + }); + const page = allPages.find((p) => p.url === id); + if (!page) { + return { content: [{ type: "text", text: "Page not found." }] }; + } + const isApiPage = page.url.startsWith("/api/"); + + const filePath = `content/${page.file.path}`; + try { + const content = await readFile(filePath, "utf-8"); + + if (isApiPage && content.includes(" { + nodeClient?.capture({ + event: "search_docs", + properties: { search_query, result_limit }, + distinctId: "mcp-handler", + }); + + const results: SearchResult[] = []; + const queryLower = search_query.toLowerCase().trim(); + const queryWords = queryLower.split(/\s+/).filter(w => w.length > 0); + + for (const page of allPages) { + if (page.url.startsWith('/api/admin/')) { + continue; + } + + let score = 0; + const title = page.data.title || ''; + const description = page.data.description || ''; + const titleLower = title.toLowerCase(); + const descriptionLower = description.toLowerCase(); + + if (titleLower.includes(queryLower)) { + if (titleLower === queryLower) { + score += 100; + } else if (titleLower.startsWith(queryLower)) { + score += 80; + } else { + score += 60; + } + } + + for (const word of queryWords) { + if (titleLower.includes(word)) { + score += 30; + } + } + + if (descriptionLower.includes(queryLower)) { + score += 40; + } + + for (const word of queryWords) { + if (descriptionLower.includes(word)) { + score += 15; + } + } + for (const tocItem of page.data.toc) { + if (typeof tocItem.title === 'string') { + const tocTitleLower = tocItem.title.toLowerCase(); + if (tocTitleLower.includes(queryLower)) { + score += 30; + } + for (const word of queryWords) { + if (tocTitleLower.includes(word)) { + score += 10; + } + } + } + } + + try { + let filePath = `content/${page.file.path}`; + if (page.url.startsWith('/api/') && !page.file.path.startsWith('api/')) { + filePath = `content/api/${page.file.path}`; + } + const content = await readFile(filePath, "utf-8"); + const textContent = content + .replace(/^---[\s\S]*?---/, '') + .replace(/<[^>]*>/g, ' ') + .replace(/\{[^}]*\}/g, ' ') + .replace(/```[a-zA-Z]*\n/g, ' ') + .replace(/```/g, ' ') + .replace(/`([^`]*)`/g, '$1') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/[#*_~]/g, '') + .replace(/\s+/g, ' ') + .trim(); + + const textContentLower = textContent.toLowerCase(); + + let hasContentMatch = false; + if (textContentLower.includes(queryLower)) { + score += 20; + hasContentMatch = true; + } + + for (const word of queryWords) { + if (textContentLower.includes(word)) { + score += 5; + hasContentMatch = true; + } + } + + if (hasContentMatch && queryWords.length > 0) { + const firstWord = queryWords[0]; + const matchIndex = textContentLower.indexOf(firstWord); + const start = Math.max(0, matchIndex - 50); + const end = Math.min(textContent.length, matchIndex + 100); + const snippet = textContent.slice(start, end); + + const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; + + results.push({ + title, + description, + url: page.url, + score, + snippet: `...${snippet}...`, + type: page.url.startsWith('/api/') ? 'api' : 'docs', + apiEndpoint + }); + } else if (score > 0) { + const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; + + results.push({ + title, + description, + url: page.url, + score, + snippet: description || title, + type: page.url.startsWith('/api/') ? 'api' : 'docs', + apiEndpoint + }); + } + } catch { + if (score > 0) { + const apiEndpoint = page.url.startsWith('/api/') ? getApiEndpointFromPage(page) : null; + + results.push({ + title, + description, + url: page.url, + score, + snippet: description || title, + type: page.url.startsWith('/api/') ? 'api' : 'docs', + apiEndpoint + }); + } + } + } + + const sortedResults = results + .sort((a, b) => b.score - a.score) + .slice(0, result_limit); + + const searchResultText = sortedResults.length > 0 + ? sortedResults.map(result => { + let text = `Title: ${result.title}\nDescription: ${result.description}\n`; + + if (result.apiEndpoint) { + text += `API Endpoint: ${result.apiEndpoint}\n`; + } + + text += `Documentation URL: ${result.url}\nType: ${result.type}\nScore: ${result.score}\nSnippet: ${result.snippet}\n`; + + return text; + }).join('\n---\n') + : `No results found for "${search_query}"`; + + return { + content: [{ type: "text", text: searchResultText }], + }; +} + +export type DocsToolAction = + | { action: "list_available_docs" } + | { action: "search_docs", search_query: string, result_limit?: number } + | { action: "get_docs_by_id", id: string } + | { action: "get_stack_auth_setup_instructions" } + | { action: "search", query: string } + | { action: "fetch", id: string }; + +export async function executeDocsToolAction(input: DocsToolAction): Promise { + switch (input.action) { + case "list_available_docs": { + nodeClient?.capture({ + event: "list_available_docs", + properties: {}, + distinctId: "mcp-handler", + }); + return { + content: [{ type: "text", text: pageSummaries }], + }; + } + case "search_docs": { + const limit = input.result_limit ?? 50; + return await searchDocsImpl(input.search_query, limit); + } + case "get_docs_by_id": { + return await getDocsByIdImpl({ id: input.id }); + } + case "get_stack_auth_setup_instructions": { + nodeClient?.capture({ + event: "get_stack_auth_setup_instructions", + properties: {}, + distinctId: "mcp-handler", + }); + + try { + const instructionsPath = path.join( + process.cwd(), + "src", + "app", + "api", + "internal", + "[transport]", + "setup-instructions.md", + ); + const instructions = await readFile(instructionsPath, "utf-8"); + + return { + content: [ + { + type: "text" as const, + text: instructions, + }, + ], + }; + } catch (error) { + return { + content: [ + { + type: "text" as const, + text: `Error reading setup instructions: ${error instanceof Error ? error.message : "Unknown error"}`, + }, + ], + isError: true, + }; + } + } + case "search": { + nodeClient?.capture({ + event: "search", + properties: { query: input.query }, + distinctId: "mcp-handler", + }); + + const q = input.query.toLowerCase(); + const results = allPages + .filter( + (page) => + page.data.title.toLowerCase().includes(q) || + page.data.description?.toLowerCase().includes(q), + ) + .map((page) => ({ + id: page.url, + title: page.data.title, + url: page.url, + })); + return { + content: [ + { + type: "text", + text: JSON.stringify({ results }), + }, + ], + }; + } + case "fetch": { + return await getDocsByIdImpl({ id: input.id }); + } + } +} diff --git a/packages/private b/packages/private index 6f708e4206..c903d22b6c 160000 --- a/packages/private +++ b/packages/private @@ -1 +1 @@ -Subproject commit 6f708e4206abc6ec0903dd93629c7bd137dbcb0b +Subproject commit c903d22b6c0688f916d16cd64c806731ee987781 From 50787475a29f3941843bd9d774d8909fa9694c43 Mon Sep 17 00:00:00 2001 From: mantrakp04 Date: Mon, 23 Mar 2026 17:13:35 -0700 Subject: [PATCH 02/51] Enhance error handling and API response for documentation tools - Introduced error capturing for failed HTTP requests in the docs tools API, improving debugging capabilities. - Updated the API response for unsupported methods to include an 'Allow' header, clarifying the expected request type. These changes enhance the robustness of the documentation tools integration and improve developer experience. --- apps/backend/src/lib/ai/tools/docs.ts | 8 ++++---- docs/src/app/api/internal/docs-tools/route.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/backend/src/lib/ai/tools/docs.ts b/apps/backend/src/lib/ai/tools/docs.ts index 37aa235d1b..3975e400a1 100644 --- a/apps/backend/src/lib/ai/tools/docs.ts +++ b/apps/backend/src/lib/ai/tools/docs.ts @@ -1,5 +1,6 @@ import { tool } from "ai"; import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { z } from "zod"; type DocsToolHttpResult = { @@ -22,11 +23,9 @@ function getDocsToolsBaseUrl(): string { async function postDocsToolAction(action: Record): Promise { const base = getDocsToolsBaseUrl(); const secret = getEnvVariable("STACK_INTERNAL_DOCS_TOOLS_SECRET", ""); - const headers: Record = { - "Content-Type": "application/json", - }; + const headers = new Headers({ "Content-Type": "application/json" }); if (secret !== "") { - headers["x-stack-internal-docs-tools-secret"] = secret; + headers.set("x-stack-internal-docs-tools-secret", secret); } const res = await fetch(`${base}/api/internal/docs-tools`, { @@ -37,6 +36,7 @@ async function postDocsToolAction(action: Record): Promise Date: Mon, 23 Mar 2026 17:36:59 -0700 Subject: [PATCH 03/51] Refactor askStackAuth key to ask_stack_auth in API documentation - Updated the key name in the capabilities section of the API documentation to follow a consistent naming convention, improving clarity and maintainability. --- docs/src/app/api/internal/[transport]/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/app/api/internal/[transport]/route.ts b/docs/src/app/api/internal/[transport]/route.ts index 05bea0edb6..945369e9a8 100644 --- a/docs/src/app/api/internal/[transport]/route.ts +++ b/docs/src/app/api/internal/[transport]/route.ts @@ -78,7 +78,7 @@ const handler = createMcpHandler( { capabilities: { tools: { - askStackAuth: { + ask_stack_auth: { description: "Ask the Stack Auth documentation assistant any question about Stack Auth (setup, APIs, SDKs, configuration, troubleshooting).", parameters: { From 274c742ecd600063131b698ea8df7b521c8d7466 Mon Sep 17 00:00:00 2001 From: mantrakp04 Date: Tue, 24 Mar 2026 20:41:06 -0700 Subject: [PATCH 04/51] fix: register private submodule gitlink in the index The .gitmodules was updated in d22593d53 to point at apps/backend/src/private/implementation, but the gitlink entry (mode 160000) was never added to the tree. This caused `git clone --recurse-submodules` to silently skip the private submodule. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/backend/src/private/implementation | 1 + 1 file changed, 1 insertion(+) create mode 160000 apps/backend/src/private/implementation diff --git a/apps/backend/src/private/implementation b/apps/backend/src/private/implementation new file mode 160000 index 0000000000..a93d7ea0c0 --- /dev/null +++ b/apps/backend/src/private/implementation @@ -0,0 +1 @@ +Subproject commit a93d7ea0c0a91d7a4dfbc97c0032c9c9c68ec4d6 From d8065c4af77f502891e7d024263c7190864ab150 Mon Sep 17 00:00:00 2001 From: mantrakp04 Date: Fri, 3 Apr 2026 17:51:10 -0700 Subject: [PATCH 05/51] Update environment configurations and remove internal secret validation for docs tools - Added `STACK_DOCS_INTERNAL_BASE_URL` to backend `.env` and `.env.development` files for AI tool bundle configuration. - Removed references to `STACK_INTERNAL_DOCS_TOOLS_SECRET` from backend and docs environment files and validation logic from the docs tools API route. - Introduced a new `.env` file for the docs app with essential configuration variables. --- apps/backend/.env | 3 +++ apps/backend/.env.development | 3 --- apps/backend/src/lib/ai/tools/docs.ts | 7 +------ docs/.env | 6 ++++++ docs/.env.development | 2 -- docs/src/app/api/internal/docs-tools/route.ts | 12 ------------ 6 files changed, 10 insertions(+), 23 deletions(-) create mode 100644 docs/.env diff --git a/apps/backend/.env b/apps/backend/.env index 1cb85405db..fc0277d588 100644 --- a/apps/backend/.env +++ b/apps/backend/.env @@ -115,3 +115,6 @@ STACK_STRIPE_SECRET_KEY=# enter your stripe api key STACK_STRIPE_WEBHOOK_SECRET=# enter your stripe webhook secret STACK_TELEGRAM_BOT_TOKEN= # enter you telegram bot token STACK_TELEGRAM_CHAT_ID=# enter your telegram chat id + +# Docs AI tool bundle +STACK_DOCS_INTERNAL_BASE_URL=# override the docs origin used by the backend's AI tool bundle to call the docs app's `/api/internal/docs-tools` endpoint. Defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod diff --git a/apps/backend/.env.development b/apps/backend/.env.development index 2ba52ced09..b2f69f3128 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -77,10 +77,7 @@ STACK_OPENAI_API_KEY=mock_openai_api_key STACK_STRIPE_SECRET_KEY=sk_test_mockstripekey STACK_STRIPE_WEBHOOK_SECRET=mock_stripe_webhook_secret STACK_OPENROUTER_API_KEY=FORWARD_TO_PRODUCTION -# Optional: override docs origin for the `docs` AI tool bundle (defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod) # STACK_DOCS_INTERNAL_BASE_URL=http://localhost:8104 -# Optional: shared secret; when set, backend sends it and docs `/api/internal/docs-tools` requires it -# STACK_INTERNAL_DOCS_TOOLS_SECRET= # Email monitor configuration for tests STACK_EMAIL_MONITOR_VERIFICATION_CALLBACK_URL=http://localhost:8101/handler/email-verification STACK_EMAIL_MONITOR_PROJECT_ID=internal diff --git a/apps/backend/src/lib/ai/tools/docs.ts b/apps/backend/src/lib/ai/tools/docs.ts index 3975e400a1..91f45ea104 100644 --- a/apps/backend/src/lib/ai/tools/docs.ts +++ b/apps/backend/src/lib/ai/tools/docs.ts @@ -22,15 +22,10 @@ function getDocsToolsBaseUrl(): string { async function postDocsToolAction(action: Record): Promise { const base = getDocsToolsBaseUrl(); - const secret = getEnvVariable("STACK_INTERNAL_DOCS_TOOLS_SECRET", ""); - const headers = new Headers({ "Content-Type": "application/json" }); - if (secret !== "") { - headers.set("x-stack-internal-docs-tools-secret", secret); - } const res = await fetch(`${base}/api/internal/docs-tools`, { method: "POST", - headers, + headers: { "Content-Type": "application/json" }, body: JSON.stringify(action), }); diff --git a/docs/.env b/docs/.env new file mode 100644 index 0000000000..2102f16e4f --- /dev/null +++ b/docs/.env @@ -0,0 +1,6 @@ +# Basic +NEXT_PUBLIC_STACK_API_URL=# the base URL of Stack's backend/API +NEXT_PUBLIC_STACK_PROJECT_ID=# the project ID to use +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=# publishable client key for the project +STACK_SECRET_SERVER_KEY=# secret server key for the project +STACK_OPENROUTER_API_KEY=# OpenRouter API key for AI-enabled chat \ No newline at end of file diff --git a/docs/.env.development b/docs/.env.development index cd0ca772f6..7f67d50ea3 100644 --- a/docs/.env.development +++ b/docs/.env.development @@ -5,5 +5,3 @@ NEXT_PUBLIC_STACK_PROJECT_ID=internal NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=this-publishable-client-key-is-for-local-development-only STACK_SECRET_SERVER_KEY=this-secret-server-key-is-for-local-development-only STACK_OPENROUTER_API_KEY=your-open-router-api-key-for-ai-enabled-chat -# Optional: require this header value for POST /api/internal/docs-tools (must match backend STACK_INTERNAL_DOCS_TOOLS_SECRET) -# STACK_INTERNAL_DOCS_TOOLS_SECRET= diff --git a/docs/src/app/api/internal/docs-tools/route.ts b/docs/src/app/api/internal/docs-tools/route.ts index 22a5594b1a..783ba1f671 100644 --- a/docs/src/app/api/internal/docs-tools/route.ts +++ b/docs/src/app/api/internal/docs-tools/route.ts @@ -15,19 +15,7 @@ const bodySchema: z.ZodType = z.discriminatedUnion("action", [ z.object({ action: z.literal("fetch"), id: z.string() }), ]); -function validateInternalSecret(req: NextRequest): boolean { - const secret = process.env.STACK_INTERNAL_DOCS_TOOLS_SECRET; - if (secret == null || secret === "") { - return true; - } - return req.headers.get("x-stack-internal-docs-tools-secret") === secret; -} - export async function POST(req: NextRequest) { - if (!validateInternalSecret(req)) { - return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - } - let json: unknown; try { json = await req.json(); From 95ca0a29618677633a24c83724e52b4eacdee8b6 Mon Sep 17 00:00:00 2001 From: Aadesh Kheria Date: Fri, 10 Apr 2026 10:16:27 -0700 Subject: [PATCH 06/51] initial commit --- README.md | 2 + apps/backend/.env | 5 + apps/backend/.env.development | 5 + apps/backend/package.json | 1 + .../app/api/latest/ai/query/[mode]/route.ts | 53 +- apps/backend/src/lib/ai/mcp-logger.ts | 51 ++ apps/backend/src/lib/ai/prompts.ts | 11 +- apps/backend/src/lib/ai/qa-reviewer.ts | 172 +++++ apps/backend/src/lib/ai/schema.ts | 6 + .../add_manual_qa_reducer.ts | 18 + .../delete_qa_entry_reducer.ts | 15 + .../src/lib/ai/spacetimedb-bindings/index.ts | 129 ++++ .../log_mcp_call_reducer.ts | 27 + .../mark_human_reviewed_reducer.ts | 16 + .../mcp_call_log_table.ts | 44 ++ .../src/lib/ai/spacetimedb-bindings/types.ts | 46 ++ .../spacetimedb-bindings/types/procedures.ts | 10 + .../ai/spacetimedb-bindings/types/reducers.ts | 22 + .../update_human_correction_reducer.ts | 19 + .../update_mcp_qa_review_reducer.ts | 25 + apps/backend/src/lib/ai/verified-qa.ts | 46 ++ apps/dev-launchpad/public/index.html | 10 + apps/internal-tool/.env | 10 + apps/internal-tool/.env.development | 8 + apps/internal-tool/.eslintrc.cjs | 7 + apps/internal-tool/next.config.mjs | 6 + apps/internal-tool/package.json | 43 ++ apps/internal-tool/postcss.config.js | 6 + apps/internal-tool/spacetime.json | 6 + apps/internal-tool/spacetimedb/package.json | 12 + apps/internal-tool/spacetimedb/src/index.ts | 224 ++++++ apps/internal-tool/spacetimedb/tsconfig.json | 12 + apps/internal-tool/src/app/app-client.tsx | 203 ++++++ apps/internal-tool/src/app/globals.css | 10 + apps/internal-tool/src/app/layout.tsx | 26 + apps/internal-tool/src/app/loading.tsx | 10 + apps/internal-tool/src/app/page.tsx | 7 + apps/internal-tool/src/app/questions/page.tsx | 74 ++ .../src/components/AddManualQa.tsx | 101 +++ .../src/components/Analytics.tsx | 261 +++++++ .../src/components/CallLogDetail.tsx | 655 ++++++++++++++++++ .../src/components/CallLogList.tsx | 314 +++++++++ .../src/components/ConversationReplay.tsx | 573 +++++++++++++++ .../src/components/KnowledgeBase.tsx | 240 +++++++ .../src/components/markdown-components.tsx | 141 ++++ .../internal-tool/src/hooks/useSpacetimeDB.ts | 113 +++ .../module_bindings/add_manual_qa_reducer.ts | 18 + .../delete_qa_entry_reducer.ts | 15 + .../src/module_bindings/index.ts | 129 ++++ .../module_bindings/log_mcp_call_reducer.ts | 27 + .../mark_human_reviewed_reducer.ts | 16 + .../src/module_bindings/mcp_call_log_table.ts | 44 ++ .../src/module_bindings/types.ts | 46 ++ .../src/module_bindings/types/procedures.ts | 10 + .../src/module_bindings/types/reducers.ts | 22 + .../update_human_correction_reducer.ts | 19 + .../update_mcp_qa_review_reducer.ts | 25 + apps/internal-tool/src/stack.ts | 20 + apps/internal-tool/src/types.ts | 1 + apps/internal-tool/src/utils.ts | 18 + apps/internal-tool/tailwind.config.js | 8 + apps/internal-tool/tsconfig.json | 43 ++ docker/dependencies/docker.compose.yaml | 9 + .../src/app/api/internal/[transport]/route.ts | 32 +- pnpm-lock.yaml | 290 ++++++-- 65 files changed, 4506 insertions(+), 81 deletions(-) create mode 100644 apps/backend/src/lib/ai/mcp-logger.ts create mode 100644 apps/backend/src/lib/ai/qa-reviewer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/index.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/types.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts create mode 100644 apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts create mode 100644 apps/backend/src/lib/ai/verified-qa.ts create mode 100644 apps/internal-tool/.env create mode 100644 apps/internal-tool/.env.development create mode 100644 apps/internal-tool/.eslintrc.cjs create mode 100644 apps/internal-tool/next.config.mjs create mode 100644 apps/internal-tool/package.json create mode 100644 apps/internal-tool/postcss.config.js create mode 100644 apps/internal-tool/spacetime.json create mode 100644 apps/internal-tool/spacetimedb/package.json create mode 100644 apps/internal-tool/spacetimedb/src/index.ts create mode 100644 apps/internal-tool/spacetimedb/tsconfig.json create mode 100644 apps/internal-tool/src/app/app-client.tsx create mode 100644 apps/internal-tool/src/app/globals.css create mode 100644 apps/internal-tool/src/app/layout.tsx create mode 100644 apps/internal-tool/src/app/loading.tsx create mode 100644 apps/internal-tool/src/app/page.tsx create mode 100644 apps/internal-tool/src/app/questions/page.tsx create mode 100644 apps/internal-tool/src/components/AddManualQa.tsx create mode 100644 apps/internal-tool/src/components/Analytics.tsx create mode 100644 apps/internal-tool/src/components/CallLogDetail.tsx create mode 100644 apps/internal-tool/src/components/CallLogList.tsx create mode 100644 apps/internal-tool/src/components/ConversationReplay.tsx create mode 100644 apps/internal-tool/src/components/KnowledgeBase.tsx create mode 100644 apps/internal-tool/src/components/markdown-components.tsx create mode 100644 apps/internal-tool/src/hooks/useSpacetimeDB.ts create mode 100644 apps/internal-tool/src/module_bindings/add_manual_qa_reducer.ts create mode 100644 apps/internal-tool/src/module_bindings/delete_qa_entry_reducer.ts create mode 100644 apps/internal-tool/src/module_bindings/index.ts create mode 100644 apps/internal-tool/src/module_bindings/log_mcp_call_reducer.ts create mode 100644 apps/internal-tool/src/module_bindings/mark_human_reviewed_reducer.ts create mode 100644 apps/internal-tool/src/module_bindings/mcp_call_log_table.ts create mode 100644 apps/internal-tool/src/module_bindings/types.ts create mode 100644 apps/internal-tool/src/module_bindings/types/procedures.ts create mode 100644 apps/internal-tool/src/module_bindings/types/reducers.ts create mode 100644 apps/internal-tool/src/module_bindings/update_human_correction_reducer.ts create mode 100644 apps/internal-tool/src/module_bindings/update_mcp_qa_review_reducer.ts create mode 100644 apps/internal-tool/src/stack.ts create mode 100644 apps/internal-tool/src/types.ts create mode 100644 apps/internal-tool/src/utils.ts create mode 100644 apps/internal-tool/tailwind.config.js create mode 100644 apps/internal-tool/tsconfig.json diff --git a/README.md b/README.md index 33388ae06b..705993615a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ [![Stack Logo](/.github/assets/logo.png)](https://stack-auth.com) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/stack-auth/stack-auth) +

📘 Docs | ☁️ Hosted Version diff --git a/apps/backend/.env b/apps/backend/.env index fc0277d588..6f28c1c4ef 100644 --- a/apps/backend/.env +++ b/apps/backend/.env @@ -118,3 +118,8 @@ STACK_TELEGRAM_CHAT_ID=# enter your telegram chat id # Docs AI tool bundle STACK_DOCS_INTERNAL_BASE_URL=# override the docs origin used by the backend's AI tool bundle to call the docs app's `/api/internal/docs-tools` endpoint. Defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod + +# MCP review tool (SpacetimeDB) +STACK_SPACETIMEDB_URI=# SpacetimeDB host URI; default empty (logging disabled) +STACK_SPACETIMEDB_DB_NAME=# SpacetimeDB database name +STACK_MCP_LOG_TOKEN=# shared secret gating the log_mcp_call reducer; must match EXPECTED_LOG_TOKEN in apps/internal-tool/spacetimedb/src/index.ts diff --git a/apps/backend/.env.development b/apps/backend/.env.development index b2f69f3128..cb90c97034 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -112,6 +112,11 @@ STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs +# MCP review tool (SpacetimeDB) +STACK_SPACETIMEDB_URI=ws://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39 +STACK_SPACETIMEDB_DB_NAME=stack-auth-llm +STACK_MCP_LOG_TOKEN=change-me + # Clickhouse STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36 STACK_CLICKHOUSE_ADMIN_USER=stackframe diff --git a/apps/backend/package.json b/apps/backend/package.json index 8d74706df2..7afecf14b6 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -55,6 +55,7 @@ }, "dependencies": { "@ai-sdk/mcp": "^1.0.21", + "spacetimedb": "^2.1.0", "@ai-sdk/openai": "^3.0.29", "@aws-sdk/client-s3": "^3.855.0", "@clickhouse/client": "^1.14.0", diff --git a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts index 829c176549..c1129b17a0 100644 --- a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts +++ b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts @@ -11,6 +11,10 @@ import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; import { Json } from "@stackframe/stack-shared/dist/utils/json"; import { generateText, ModelMessage, stepCountIs, streamText } from "ai"; +import { logMcpCall } from "@/lib/ai/mcp-logger"; +import { reviewMcpCall } from "@/lib/ai/qa-reviewer"; +import { getVerifiedQaContext } from "@/lib/ai/verified-qa"; +import { runAsynchronously } from "@stackframe/stack-shared/dist/utils/promises"; export const POST = createSmartRouteHandler({ metadata: { @@ -61,10 +65,13 @@ export const POST = createSmartRouteHandler({ } const model = selectModel(quality, speed, isAuthenticated); - const systemPrompt = getFullSystemPrompt(systemPromptId); + const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; + let systemPrompt = getFullSystemPrompt(systemPromptId); + if (isDocsOrSearch) { + systemPrompt += await getVerifiedQaContext(); + } const tools = await getTools(toolNames, { auth: fullReq.auth, targetProjectId: projectId }); const toolsArg = Object.keys(tools).length > 0 ? tools : undefined; - const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; const stepLimit = toolsArg == null ? 1 : isDocsOrSearch ? 50 : 5; if (mode === "stream") { @@ -81,6 +88,7 @@ export const POST = createSmartRouteHandler({ body: result.toUIMessageStreamResponse(), }; } else { + const startedAt = Date.now(); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 120_000); const result = await generateText({ @@ -129,10 +137,49 @@ export const POST = createSmartRouteHandler({ }); }); + let responseConversationId: string | undefined; + if (body.mcpCallMetadata != null) { + const correlationId = crypto.randomUUID(); + const conversationId = body.mcpCallMetadata.conversationId ?? crypto.randomUUID(); + responseConversationId = conversationId; + const firstUserMessage = messages.find(m => m.role === "user"); + const question = typeof firstUserMessage?.content === "string" + ? firstUserMessage.content + : JSON.stringify(firstUserMessage?.content ?? ""); + + const innerToolCallsJson = JSON.stringify(contentBlocks.filter(b => b.type === "tool-call")); + + runAsynchronously(logMcpCall({ + correlationId, + toolName: body.mcpCallMetadata.toolName, + reason: body.mcpCallMetadata.reason, + userPrompt: body.mcpCallMetadata.userPrompt, + conversationId, + question, + response: result.text, + stepCount: result.steps.length, + innerToolCallsJson, + durationMs: BigInt(Date.now() - startedAt), + modelId: String(model.modelId), + errorMessage: undefined, + })); + + runAsynchronously(reviewMcpCall({ + correlationId, + question, + reason: body.mcpCallMetadata.reason, + response: result.text, + })); + } + return { statusCode: 200, bodyType: "json" as const, - body: { content: contentBlocks, finalText: result.text }, + body: { + content: contentBlocks, + finalText: result.text, + conversationId: responseConversationId ?? null, + }, }; } }, diff --git a/apps/backend/src/lib/ai/mcp-logger.ts b/apps/backend/src/lib/ai/mcp-logger.ts new file mode 100644 index 0000000000..97ebb25241 --- /dev/null +++ b/apps/backend/src/lib/ai/mcp-logger.ts @@ -0,0 +1,51 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { DbConnection } from "./spacetimedb-bindings"; +import type { LogMcpCallParams } from "./spacetimedb-bindings/types/reducers"; + +export type McpLogEntry = Omit; + +let connectionPromise: Promise | null = null; + +export async function getConnection(): Promise { + const uri = getEnvVariable("STACK_SPACETIMEDB_URI", ""); + if (!uri) { + return null; + } + + if (!connectionPromise) { + connectionPromise = new Promise((resolve, reject) => { + DbConnection.builder() + .withUri(uri) + .withDatabaseName(getEnvVariable("STACK_SPACETIMEDB_DB_NAME")) + .onConnect((connInstance) => { + connInstance.subscriptionBuilder() + .onApplied(() => { + resolve(connInstance); + }) + .subscribe("SELECT * FROM mcp_call_log"); + }) + .onConnectError((_: unknown, err: Error) => { + captureError("mcp-logger", err); + connectionPromise = null; + reject(err); + }) + .build(); + }); + } + + return await connectionPromise; +} + +export async function logMcpCall(entry: McpLogEntry): Promise { + const conn = await getConnection(); + if (!conn) { + return; + } + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.logMcpCall({ + token, + ...entry, + }); +} diff --git a/apps/backend/src/lib/ai/prompts.ts b/apps/backend/src/lib/ai/prompts.ts index 1ce70b5af1..59cece449f 100644 --- a/apps/backend/src/lib/ai/prompts.ts +++ b/apps/backend/src/lib/ai/prompts.ts @@ -112,11 +112,12 @@ You are Stack Auth's AI assistant. You help users with Stack Auth - a complete a Think step by step about what to say. Being wrong is 100x worse than saying you don't know. -## TOOL USAGE WORKFLOW: -1. **FIRST**, use \`search_docs\` with relevant keywords to find related documentation -2. **THEN**, use \`get_docs_by_id\` to retrieve the full content of the most relevant pages -3. Base your answer on the actual documentation content retrieved -4. When referring to API endpoints, **always cite the actual endpoint** (e.g., "GET /users/me") not the documentation URL +## PRIORITY ORDER: +1. **FIRST**, check the Human-Verified Knowledge Base (appended at the end of this prompt, if any). If the user's question matches or is similar to a verified Q&A, use that answer exactly — do not search docs or use any other source. +2. **THEN**, use \`search_docs\` with relevant keywords to find related documentation +3. **THEN**, use \`get_docs_by_id\` to retrieve the full content of the most relevant pages +4. Base your answer on the actual documentation content retrieved +5. When referring to API endpoints, **always cite the actual endpoint** (e.g., "GET /users/me") not the documentation URL ## CORE RESPONSIBILITIES: 1. Help users implement Stack Auth in their applications diff --git a/apps/backend/src/lib/ai/qa-reviewer.ts b/apps/backend/src/lib/ai/qa-reviewer.ts new file mode 100644 index 0000000000..e8c817a61d --- /dev/null +++ b/apps/backend/src/lib/ai/qa-reviewer.ts @@ -0,0 +1,172 @@ +import { createMCPClient } from "@ai-sdk/mcp"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { generateText, stepCountIs } from "ai"; +import { getConnection } from "./mcp-logger"; +import { createOpenRouterProvider } from "./models"; +import { getVerifiedQaContext } from "./verified-qa"; + +const QA_SYSTEM_PROMPT = `You are a QA reviewer for Stack Auth's AI documentation assistant. +You will receive a question, the agent's stated reason for asking, and the AI's response. + +Your tasks: +1. RELEVANCE: Does the response actually answer the question? Does the stated reason align with what was asked? +2. CORRECTNESS: Verify factual claims about Stack Auth. Use human-verified Q&A (appended below, if any) as the highest-priority source of truth — these are always correct. Then use the available tools to look up additional information from the Stack Auth codebase. If the AI response contradicts a human-verified answer, flag it as incorrect. + +The repo name for all tool calls is "stack-auth/stack-auth". Only use the repository documentation tools (read_wiki_structure, read_wiki_contents, ask_question) — do not create sessions or modify any other resources. + +You MUST respond with ONLY valid JSON matching this exact schema (no markdown, no explanation outside the JSON): +{ + "needsHumanReview": boolean, + "answerCorrect": boolean, + "answerRelevant": boolean, + "flags": [{"type": string, "severity": "low" | "medium" | "high" | "critical", "explanation": string}], + "improvementSuggestions": string, + "overallScore": number +} + +Flag types: "factual_error", "incomplete_answer", "off_topic", "hallucination", "outdated_info", "missing_context", "misleading", "reason_mismatch" + +Scoring: +- 90-100: Excellent — factually correct, fully addresses the question +- 70-89: Good — minor issues or missing details +- 50-69: Acceptable — notable issues but core answer is present +- 30-49: Poor — significant problems +- 0-29: Unacceptable — fundamentally wrong or irrelevant + +Set needsHumanReview=true if: score < 50, any critical flag, or you are uncertain about correctness.`; + +const REVIEW_MODEL_ID = "anthropic/claude-haiku-4.5"; + +export async function reviewMcpCall(entry: { + correlationId: string; + question: string; + reason: string; + response: string; +}): Promise { + const apiKey = getEnvVariable("STACK_OPENROUTER_API_KEY", ""); + if (!apiKey || apiKey === "FORWARD_TO_PRODUCTION") { + return; + } + + let devinClient: Awaited> | null = null; + + const failureUpdate = (err: unknown) => ({ + qaNeedsHumanReview: true, + qaAnswerCorrect: false, + qaAnswerRelevant: false, + qaFlagsJson: "[]", + qaImprovementSuggestions: "", + qaOverallScore: 0, + qaConversationJson: undefined, + qaErrorMessage: String(err), + }); + + let update: { + qaNeedsHumanReview: boolean, + qaAnswerCorrect: boolean, + qaAnswerRelevant: boolean, + qaFlagsJson: string, + qaImprovementSuggestions: string, + qaOverallScore: number, + qaConversationJson: string | undefined, + qaErrorMessage: string | undefined, + }; + + try { + // Wait for the log row to be written first + await new Promise(r => setTimeout(r, 3000)); + + devinClient = await createMCPClient({ + transport: { + type: "http", + url: "https://mcp.deepwiki.com/mcp", + }, + }); + + const devinTools = await devinClient.tools(); + const openrouter = createOpenRouterProvider(); + const model = openrouter(REVIEW_MODEL_ID); + + const userMessage = [ + "## Question", + entry.question, + "", + "## Agent's Reason for Asking", + entry.reason, + "", + "## AI Response", + entry.response, + ].join("\n"); + + const verifiedQa = await getVerifiedQaContext(); + + const result = await generateText({ + model, + system: QA_SYSTEM_PROMPT + verifiedQa, + tools: devinTools as Parameters[0]["tools"], + stopWhen: stepCountIs(10), + messages: [{ role: "user", content: userMessage }], + }); + + const conversation = result.steps.map((step, i) => { + const toolCalls = step.toolCalls.map(tc => ({ toolName: tc.toolName, args: tc.input })); + const toolResults = step.toolResults.map(tr => ({ + toolName: tr.toolName, + toolCallId: tr.toolCallId, + result: tr.output, + })); + return { + step: i + 1, + text: step.text || undefined, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + }; + }); + + const jsonMatch = result.text.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error("No JSON found in QA review response"); + } + const parsed = JSON.parse(jsonMatch[0]) as { + needsHumanReview: boolean, + answerCorrect: boolean, + answerRelevant: boolean, + flags: Array<{ type: string, severity: string, explanation: string }>, + improvementSuggestions: string, + overallScore: number, + }; + + update = { + qaNeedsHumanReview: parsed.needsHumanReview, + qaAnswerCorrect: parsed.answerCorrect, + qaAnswerRelevant: parsed.answerRelevant, + qaFlagsJson: JSON.stringify(parsed.flags), + qaImprovementSuggestions: parsed.improvementSuggestions, + qaOverallScore: parsed.overallScore, + qaConversationJson: JSON.stringify(conversation), + qaErrorMessage: undefined, + }; + } catch (err) { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + update = failureUpdate(err); + } + + if (devinClient) { + await devinClient.close().catch((err: unknown) => { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + }); + } + + const conn = await getConnection(); + if (!conn) return; + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.updateMcpQaReview({ + token, + correlationId: entry.correlationId, + qaReviewModelId: REVIEW_MODEL_ID, + ...update, + }).catch((err: unknown) => { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + }); +} diff --git a/apps/backend/src/lib/ai/schema.ts b/apps/backend/src/lib/ai/schema.ts index e27a5abe27..27e73c8224 100644 --- a/apps/backend/src/lib/ai/schema.ts +++ b/apps/backend/src/lib/ai/schema.ts @@ -24,6 +24,12 @@ export const requestBodySchema = yupObject({ }).defined() ).defined().min(1), projectId: yupString().optional().nullable(), + mcpCallMetadata: yupObject({ + toolName: yupString().defined(), + reason: yupString().defined(), + userPrompt: yupString().defined(), + conversationId: yupString().optional().nullable(), + }).optional().nullable(), }); export type RequestBody = InferType; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts new file mode 100644 index 0000000000..79e578f933 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + question: __t.string(), + answer: __t.string(), + publish: __t.bool(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts new file mode 100644 index 0000000000..11f22a34e9 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts @@ -0,0 +1,15 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + correlationId: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts new file mode 100644 index 0000000000..bf16c63e07 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts @@ -0,0 +1,129 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb cli version 2.1.0 (commit 6981f48b4bc1a71c8dd9bdfe5a2c343f6370243d). + +/* eslint-disable */ +/* tslint:disable */ +import { + DbConnectionBuilder as __DbConnectionBuilder, + DbConnectionImpl as __DbConnectionImpl, + SubscriptionBuilderImpl as __SubscriptionBuilderImpl, + TypeBuilder as __TypeBuilder, + Uuid as __Uuid, + convertToAccessorMap as __convertToAccessorMap, + makeQueryBuilder as __makeQueryBuilder, + procedureSchema as __procedureSchema, + procedures as __procedures, + reducerSchema as __reducerSchema, + reducers as __reducers, + schema as __schema, + t as __t, + table as __table, + type AlgebraicTypeType as __AlgebraicTypeType, + type DbConnectionConfig as __DbConnectionConfig, + type ErrorContextInterface as __ErrorContextInterface, + type Event as __Event, + type EventContextInterface as __EventContextInterface, + type Infer as __Infer, + type QueryBuilder as __QueryBuilder, + type ReducerEventContextInterface as __ReducerEventContextInterface, + type RemoteModule as __RemoteModule, + type SubscriptionEventContextInterface as __SubscriptionEventContextInterface, + type SubscriptionHandleImpl as __SubscriptionHandleImpl, +} from "spacetimedb"; + +// Import all reducer arg schemas +import AddManualQaReducer from "./add_manual_qa_reducer"; +import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; +import LogMcpCallReducer from "./log_mcp_call_reducer"; +import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; +import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; +import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; + +// Import all procedure arg schemas + +// Import all table schema definitions +import McpCallLogRow from "./mcp_call_log_table"; + +/** Type-only namespace exports for generated type groups. */ + +/** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ +const tablesSchema = __schema({ + mcpCallLog: __table({ + name: 'mcp_call_log', + indexes: [ + { accessor: 'id', name: 'mcp_call_log_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'mcp_call_log_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, McpCallLogRow), +}); + +/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ +const reducersSchema = __reducers( + __reducerSchema("add_manual_qa", AddManualQaReducer), + __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), + __reducerSchema("log_mcp_call", LogMcpCallReducer), + __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), + __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), + __reducerSchema("update_mcp_qa_review", UpdateMcpQaReviewReducer), +); + +/** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ +const proceduresSchema = __procedures( +); + +/** The remote SpacetimeDB module schema, both runtime and type information. */ +const REMOTE_MODULE = { + versionInfo: { + cliVersion: "2.1.0" as const, + }, + tables: tablesSchema.schemaType.tables, + reducers: reducersSchema.reducersType.reducers, + ...proceduresSchema, +} satisfies __RemoteModule< + typeof tablesSchema.schemaType, + typeof reducersSchema.reducersType, + typeof proceduresSchema +>; + +/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */ +export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType); + +/** The reducers available in this remote SpacetimeDB module. */ +export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers); + +/** The context type returned in callbacks for all possible events. */ +export type EventContext = __EventContextInterface; +/** The context type returned in callbacks for reducer events. */ +export type ReducerEventContext = __ReducerEventContextInterface; +/** The context type returned in callbacks for subscription events. */ +export type SubscriptionEventContext = __SubscriptionEventContextInterface; +/** The context type returned in callbacks for error events. */ +export type ErrorContext = __ErrorContextInterface; +/** The subscription handle type to manage active subscriptions created from a {@link SubscriptionBuilder}. */ +export type SubscriptionHandle = __SubscriptionHandleImpl; + +/** Builder class to configure a new subscription to the remote SpacetimeDB instance. */ +export class SubscriptionBuilder extends __SubscriptionBuilderImpl {} + +/** Builder class to configure a new database connection to the remote SpacetimeDB instance. */ +export class DbConnectionBuilder extends __DbConnectionBuilder {} + +/** The typed database connection to manage connections to the remote SpacetimeDB instance. This class has type information specific to the generated module. */ +export class DbConnection extends __DbConnectionImpl { + /** Creates a new {@link DbConnectionBuilder} to configure and connect to the remote SpacetimeDB instance. */ + static builder = (): DbConnectionBuilder => { + return new DbConnectionBuilder(REMOTE_MODULE, (config: __DbConnectionConfig) => new DbConnection(config)); + }; + + /** Creates a new {@link SubscriptionBuilder} to configure a subscription to the remote SpacetimeDB instance. */ + override subscriptionBuilder = (): SubscriptionBuilder => { + return new SubscriptionBuilder(this); + }; +} + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts new file mode 100644 index 0000000000..772cd1fc78 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts @@ -0,0 +1,27 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + conversationId: __t.option(__t.string()), + toolName: __t.string(), + reason: __t.string(), + userPrompt: __t.string(), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32(), + innerToolCallsJson: __t.string(), + durationMs: __t.u64(), + modelId: __t.string(), + errorMessage: __t.option(__t.string()), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts new file mode 100644 index 0000000000..ac2547b4d2 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + correlationId: __t.string(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts new file mode 100644 index 0000000000..bcf0dab879 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts @@ -0,0 +1,44 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + correlationId: __t.string().name("correlation_id"), + conversationId: __t.option(__t.string()).name("conversation_id"), + createdAt: __t.timestamp().name("created_at"), + toolName: __t.string().name("tool_name"), + reason: __t.string(), + userPrompt: __t.string().name("user_prompt"), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32().name("step_count"), + innerToolCallsJson: __t.string().name("inner_tool_calls_json"), + durationMs: __t.u64().name("duration_ms"), + modelId: __t.string().name("model_id"), + errorMessage: __t.option(__t.string()).name("error_message"), + qaReviewedAt: __t.option(__t.timestamp()).name("qa_reviewed_at"), + qaNeedsHumanReview: __t.option(__t.bool()).name("qa_needs_human_review"), + qaAnswerCorrect: __t.option(__t.bool()).name("qa_answer_correct"), + qaAnswerRelevant: __t.option(__t.bool()).name("qa_answer_relevant"), + qaFlagsJson: __t.option(__t.string()).name("qa_flags_json"), + qaImprovementSuggestions: __t.option(__t.string()).name("qa_improvement_suggestions"), + qaOverallScore: __t.option(__t.u32()).name("qa_overall_score"), + qaReviewModelId: __t.option(__t.string()).name("qa_review_model_id"), + qaConversationJson: __t.option(__t.string()).name("qa_conversation_json"), + qaErrorMessage: __t.option(__t.string()).name("qa_error_message"), + humanReviewedAt: __t.option(__t.timestamp()).name("human_reviewed_at"), + humanReviewedBy: __t.option(__t.string()).name("human_reviewed_by"), + humanCorrectedQuestion: __t.option(__t.string()).name("human_corrected_question"), + humanCorrectedAnswer: __t.option(__t.string()).name("human_corrected_answer"), + publishedToQa: __t.option(__t.bool()).name("published_to_qa"), + publishedAt: __t.option(__t.timestamp()).name("published_at"), +}); diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts new file mode 100644 index 0000000000..4af9e7b736 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts @@ -0,0 +1,46 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const McpCallLog = __t.object("McpCallLog", { + id: __t.u64(), + correlationId: __t.string(), + conversationId: __t.option(__t.string()), + createdAt: __t.timestamp(), + toolName: __t.string(), + reason: __t.string(), + userPrompt: __t.string(), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32(), + innerToolCallsJson: __t.string(), + durationMs: __t.u64(), + modelId: __t.string(), + errorMessage: __t.option(__t.string()), + qaReviewedAt: __t.option(__t.timestamp()), + qaNeedsHumanReview: __t.option(__t.bool()), + qaAnswerCorrect: __t.option(__t.bool()), + qaAnswerRelevant: __t.option(__t.bool()), + qaFlagsJson: __t.option(__t.string()), + qaImprovementSuggestions: __t.option(__t.string()), + qaOverallScore: __t.option(__t.u32()), + qaReviewModelId: __t.option(__t.string()), + qaConversationJson: __t.option(__t.string()), + qaErrorMessage: __t.option(__t.string()), + humanReviewedAt: __t.option(__t.timestamp()), + humanReviewedBy: __t.option(__t.string()), + humanCorrectedQuestion: __t.option(__t.string()), + humanCorrectedAnswer: __t.option(__t.string()), + publishedToQa: __t.option(__t.bool()), + publishedAt: __t.option(__t.timestamp()), +}); +export type McpCallLog = __Infer; + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts new file mode 100644 index 0000000000..d5ac825c9a --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts @@ -0,0 +1,10 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all procedure arg schemas + + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts new file mode 100644 index 0000000000..87a6606ae4 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts @@ -0,0 +1,22 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all reducer arg schemas +import AddManualQaReducer from "../add_manual_qa_reducer"; +import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; +import LogMcpCallReducer from "../log_mcp_call_reducer"; +import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; +import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; +import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; + +export type AddManualQaParams = __Infer; +export type DeleteQaEntryParams = __Infer; +export type LogMcpCallParams = __Infer; +export type MarkHumanReviewedParams = __Infer; +export type UpdateHumanCorrectionParams = __Infer; +export type UpdateMcpQaReviewParams = __Infer; + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts new file mode 100644 index 0000000000..c026d35162 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + correlationId: __t.string(), + correctedQuestion: __t.string(), + correctedAnswer: __t.string(), + publish: __t.bool(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts new file mode 100644 index 0000000000..035e1d017c --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts @@ -0,0 +1,25 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + qaNeedsHumanReview: __t.bool(), + qaAnswerCorrect: __t.bool(), + qaAnswerRelevant: __t.bool(), + qaFlagsJson: __t.string(), + qaImprovementSuggestions: __t.string(), + qaOverallScore: __t.u32(), + qaReviewModelId: __t.string(), + qaConversationJson: __t.option(__t.string()), + qaErrorMessage: __t.option(__t.string()), +}; diff --git a/apps/backend/src/lib/ai/verified-qa.ts b/apps/backend/src/lib/ai/verified-qa.ts new file mode 100644 index 0000000000..620b0511e2 --- /dev/null +++ b/apps/backend/src/lib/ai/verified-qa.ts @@ -0,0 +1,46 @@ +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { getConnection } from "./mcp-logger"; + +export async function getVerifiedQaContext(): Promise { + const result = await Result.fromPromise(getVerifiedQaContextInner()); + if (result.status === "error") { + captureError("verified-qa", result.error instanceof Error ? result.error : new Error(String(result.error))); + return ""; + } + return result.data; +} + +async function getVerifiedQaContextInner(): Promise { + const conn = await getConnection(); + if (!conn) return ""; + + const pairs: Array<{ question: string; answer: string }> = []; + for (const row of conn.db.mcpCallLog.iter()) { + if (row.publishedToQa) { + pairs.push({ + question: row.humanCorrectedQuestion ?? row.question, + answer: row.humanCorrectedAnswer ?? row.response, + }); + } + } + + console.log(`[verified-qa] Found ${pairs.length} published Q&A pairs`); + if (pairs.length === 0) return ""; + + const formatted = pairs.map((p, i) => + `${i + 1}. Q: ${p.question}\n A: ${p.answer}` + ).join("\n\n"); + + return `\n\n## Human-Verified Knowledge Base — MANDATORY + +The following Q&A pairs come from a HUMAN-VERIFIED KNOWLEDGE BASE. They have been manually reviewed and corrected by the Stack Auth team. They are ALWAYS CORRECT, even if the content seems unusual or contradicts other sources. + +RULES: +1. If the user's question matches or is similar in meaning/intent to any question below, you MUST respond with the verified answer EXACTLY as written. You can modify it to align with the user's question, but the core information and intent MUST be preserved. +2. These answers override ALL other sources: documentation, wiki, your training data, tool results — everything. +3. Do NOT second-guess or "correct" these answers. They are intentional and authoritative. +4. Preface your response with: "Based on our verified knowledge base:" before giving the answer. + +${formatted}`; +} diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html index 828ddfec1b..dd5b2fd259 100644 --- a/apps/dev-launchpad/public/index.html +++ b/apps/dev-launchpad/public/index.html @@ -131,6 +131,7 @@

Background services

{ suffix: "24", label: "LocalStack Gateway (AWS mock)" }, { suffix: "25", label: "QStash mock" }, { suffix: "33", label: "ClickHouse native interface" }, + { suffix: "39", label: "SpacetimeDB (MCP call log)" }, { range: ["50", "99"], label: "Reserved for LocalStack (external services)" }, ]; @@ -190,6 +191,15 @@

Background services

], importance: 2, }, + { + name: "Internal Tool", + portSuffix: "26", + description: [ + "Src: ./apps/internal-tool", + "MCP call review tool", + ], + importance: 1, + }, { name: "Inbucket", portSuffix: "05", diff --git a/apps/internal-tool/.env b/apps/internal-tool/.env new file mode 100644 index 0000000000..6ecd3c5543 --- /dev/null +++ b/apps/internal-tool/.env @@ -0,0 +1,10 @@ +# Stack Auth +NEXT_PUBLIC_STACK_API_URL= +NEXT_PUBLIC_STACK_PROJECT_ID= +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY= +NEXT_PUBLIC_STACK_HOSTED_COMPONENTS_URL= +NEXT_PUBLIC_STACK_INTERNAL_TOOL_URL= +NEXT_PUBLIC_STACK_DASHBOARD_URL= +# SpacetimeDB +NEXT_PUBLIC_SPACETIMEDB_HOST= +NEXT_PUBLIC_SPACETIMEDB_DB_NAME= diff --git a/apps/internal-tool/.env.development b/apps/internal-tool/.env.development new file mode 100644 index 0000000000..1fb0a15f4f --- /dev/null +++ b/apps/internal-tool/.env.development @@ -0,0 +1,8 @@ +NEXT_PUBLIC_STACK_API_URL=http://localhost:8102 +NEXT_PUBLIC_STACK_PROJECT_ID=internal +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=this-publishable-client-key-is-for-local-development-only +NEXT_PUBLIC_STACK_HOSTED_COMPONENTS_URL=http://internal.localhost:8109 +NEXT_PUBLIC_STACK_INTERNAL_TOOL_URL=http://localhost:8126 +NEXT_PUBLIC_STACK_DASHBOARD_URL=http://localhost:8101 +NEXT_PUBLIC_SPACETIMEDB_HOST=ws://localhost:8139 +NEXT_PUBLIC_SPACETIMEDB_DB_NAME=stack-auth-llm diff --git a/apps/internal-tool/.eslintrc.cjs b/apps/internal-tool/.eslintrc.cjs new file mode 100644 index 0000000000..f0b54a7474 --- /dev/null +++ b/apps/internal-tool/.eslintrc.cjs @@ -0,0 +1,7 @@ +module.exports = { + extends: ["../../configs/eslint/defaults.js"], + parserOptions: { + project: "./tsconfig.json", + }, + ignorePatterns: ["/*", "!/src"], +}; diff --git a/apps/internal-tool/next.config.mjs b/apps/internal-tool/next.config.mjs new file mode 100644 index 0000000000..575cb88e2e --- /dev/null +++ b/apps/internal-tool/next.config.mjs @@ -0,0 +1,6 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + poweredByHeader: false, +}; + +export default nextConfig; diff --git a/apps/internal-tool/package.json b/apps/internal-tool/package.json new file mode 100644 index 0000000000..859f68e883 --- /dev/null +++ b/apps/internal-tool/package.json @@ -0,0 +1,43 @@ +{ + "name": "@stackframe/internal-tool", + "private": true, + "version": "2.8.80", + "type": "module", + "scripts": { + "dev": "pnpm spacetime:publish:local && next dev --turbopack --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}26", + "build": "next build", + "start": "next start --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}26", + "typecheck": "tsc --noEmit", + "lint": "eslint --ext .ts,.tsx .", + "clean": "rimraf .next && rimraf node_modules", + "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", + "spacetime:inject-token": "sed -i.bak \"s/__SPACETIMEDB_LOG_TOKEN__/${STACK_MCP_LOG_TOKEN:-change-me}/\" spacetimedb/src/index.ts", + "spacetime:restore-token": "mv spacetimedb/src/index.ts.bak spacetimedb/src/index.ts", + "spacetime:publish:local": "pnpm spacetime:inject-token && spacetime publish stack-auth-llm --server local -p spacetimedb --yes --no-config --delete-data=on-conflict; EXIT=$?; pnpm spacetime:restore-token; exit $EXIT", + "spacetime:publish:prod": "pnpm spacetime:inject-token && spacetime publish stack-auth-llm --server maincloud -p spacetimedb --yes --no-config; EXIT=$?; pnpm spacetime:restore-token; exit $EXIT" + }, + "dependencies": { + "@stackframe/react": "workspace:*", + "@stackframe/stack-shared": "workspace:*", + "clsx": "^2.1.1", + "date-fns": "^4.1.0", + "next": "16.1.5", + "react-markdown": "^10.1.0", + "zod": "^3.24.0", + "remark-gfm": "^4.0.1", + "react": "19.2.3", + "react-dom": "19.2.3", + "spacetimedb": "^2.1.0", + "tailwind-merge": "^3.3.1" + }, + "devDependencies": { + "@types/react": "19.2.7", + "@types/react-dom": "19.2.3", + "autoprefixer": "^10.4.21", + "postcss": "^8.5.4", + "@tailwindcss/typography": "^0.5.16", + "tailwindcss": "^3.4.17", + "typescript": "5.9.3" + }, + "packageManager": "pnpm@10.23.0" +} diff --git a/apps/internal-tool/postcss.config.js b/apps/internal-tool/postcss.config.js new file mode 100644 index 0000000000..2aa7205d4b --- /dev/null +++ b/apps/internal-tool/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/apps/internal-tool/spacetime.json b/apps/internal-tool/spacetime.json new file mode 100644 index 0000000000..11f5588437 --- /dev/null +++ b/apps/internal-tool/spacetime.json @@ -0,0 +1,6 @@ +{ + "dev": { + "run": "npm run dev" + }, + "module_path": "spacetimedb" +} \ No newline at end of file diff --git a/apps/internal-tool/spacetimedb/package.json b/apps/internal-tool/spacetimedb/package.json new file mode 100644 index 0000000000..5b11f15b53 --- /dev/null +++ b/apps/internal-tool/spacetimedb/package.json @@ -0,0 +1,12 @@ +{ + "name": "@stackframe/internal-tool-module", + "private": true, + "version": "2.8.80", + "type": "module", + "dependencies": { + "spacetimedb": "^2.1.0" + }, + "devDependencies": { + "typescript": "5.9.3" + } +} diff --git a/apps/internal-tool/spacetimedb/src/index.ts b/apps/internal-tool/spacetimedb/src/index.ts new file mode 100644 index 0000000000..fc65e5d6d1 --- /dev/null +++ b/apps/internal-tool/spacetimedb/src/index.ts @@ -0,0 +1,224 @@ +import { schema, t, table, SenderError } from 'spacetimedb/server'; + +// Injected at publish time by the spacetime:inject-token pnpm script from STACK_MCP_LOG_TOKEN env var. +// Must match STACK_MCP_LOG_TOKEN in the backend .env. +const EXPECTED_LOG_TOKEN = '__SPACETIMEDB_LOG_TOKEN__'; + +const mcpCallLog = table( + { name: 'mcp_call_log', public: true }, + { + id: t.u64().primaryKey().autoInc(), + correlationId: t.string(), + conversationId: t.string().optional(), + createdAt: t.timestamp(), + toolName: t.string(), + reason: t.string(), + userPrompt: t.string(), + question: t.string(), + response: t.string(), + stepCount: t.u32(), + innerToolCallsJson: t.string(), + durationMs: t.u64(), + modelId: t.string(), + errorMessage: t.string().optional(), + // QA review fields (populated asynchronously after initial log) + qaReviewedAt: t.timestamp().optional(), + qaNeedsHumanReview: t.bool().optional(), + qaAnswerCorrect: t.bool().optional(), + qaAnswerRelevant: t.bool().optional(), + qaFlagsJson: t.string().optional(), + qaImprovementSuggestions: t.string().optional(), + qaOverallScore: t.u32().optional(), + qaReviewModelId: t.string().optional(), + qaConversationJson: t.string().optional(), + qaErrorMessage: t.string().optional(), + // Human review + humanReviewedAt: t.timestamp().optional(), + humanReviewedBy: t.string().optional(), + // Human corrections & publishing + humanCorrectedQuestion: t.string().optional(), + humanCorrectedAnswer: t.string().optional(), + publishedToQa: t.bool().optional(), + publishedAt: t.timestamp().optional(), + } +); + +const spacetimedb = schema({ mcpCallLog }); +export default spacetimedb; + +export const log_mcp_call = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + conversationId: t.string().optional(), + toolName: t.string(), + reason: t.string(), + userPrompt: t.string(), + question: t.string(), + response: t.string(), + stepCount: t.u32(), + innerToolCallsJson: t.string(), + durationMs: t.u64(), + modelId: t.string(), + errorMessage: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.mcpCallLog.insert({ + id: 0n, + correlationId: args.correlationId, + conversationId: args.conversationId, + createdAt: ctx.timestamp, + toolName: args.toolName, + reason: args.reason, + userPrompt: args.userPrompt, + question: args.question, + response: args.response, + stepCount: args.stepCount, + innerToolCallsJson: args.innerToolCallsJson, + durationMs: args.durationMs, + modelId: args.modelId, + errorMessage: args.errorMessage, + } as Parameters[0]); + } +); + +export const update_mcp_qa_review = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + qaNeedsHumanReview: t.bool(), + qaAnswerCorrect: t.bool(), + qaAnswerRelevant: t.bool(), + qaFlagsJson: t.string(), + qaImprovementSuggestions: t.string(), + qaOverallScore: t.u32(), + qaReviewModelId: t.string(), + qaConversationJson: t.string().optional(), + qaErrorMessage: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + qaReviewedAt: ctx.timestamp, + qaNeedsHumanReview: args.qaNeedsHumanReview, + qaAnswerCorrect: args.qaAnswerCorrect, + qaAnswerRelevant: args.qaAnswerRelevant, + qaFlagsJson: args.qaFlagsJson, + qaImprovementSuggestions: args.qaImprovementSuggestions, + qaOverallScore: args.qaOverallScore, + qaReviewModelId: args.qaReviewModelId, + qaConversationJson: args.qaConversationJson, + qaErrorMessage: args.qaErrorMessage, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const mark_human_reviewed = spacetimedb.reducer( + { + correlationId: t.string(), + reviewedBy: t.string(), + }, + (ctx, args) => { + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + humanReviewedAt: ctx.timestamp, + humanReviewedBy: args.reviewedBy, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const update_human_correction = spacetimedb.reducer( + { + correlationId: t.string(), + correctedQuestion: t.string(), + correctedAnswer: t.string(), + publish: t.bool(), + reviewedBy: t.string(), + }, + (ctx, args) => { + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + humanCorrectedQuestion: args.correctedQuestion, + humanCorrectedAnswer: args.correctedAnswer, + humanReviewedAt: row.humanReviewedAt ?? ctx.timestamp, + humanReviewedBy: row.humanReviewedBy ?? args.reviewedBy, + publishedToQa: args.publish, + publishedAt: args.publish ? (row.publishedAt ?? ctx.timestamp) : row.publishedAt, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const add_manual_qa = spacetimedb.reducer( + { + question: t.string(), + answer: t.string(), + publish: t.bool(), + reviewedBy: t.string(), + }, + (ctx, args) => { + ctx.db.mcpCallLog.insert({ + id: 0n, + correlationId: ctx.newUuidV4().toString(), + createdAt: ctx.timestamp, + toolName: "manual", + reason: "Manually added Q&A", + userPrompt: "", + question: args.question, + response: "", + stepCount: 0, + innerToolCallsJson: "[]", + durationMs: 0n, + modelId: "human", + humanCorrectedQuestion: args.question, + humanCorrectedAnswer: args.answer, + humanReviewedAt: ctx.timestamp, + humanReviewedBy: args.reviewedBy, + publishedToQa: args.publish, + publishedAt: args.publish ? ctx.timestamp : undefined, + } as Parameters[0]); + } +); + +export const delete_qa_entry = spacetimedb.reducer( + { + correlationId: t.string(), + }, + (ctx, args) => { + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const init = spacetimedb.init(_ctx => {}); diff --git a/apps/internal-tool/spacetimedb/tsconfig.json b/apps/internal-tool/spacetimedb/tsconfig.json new file mode 100644 index 0000000000..93673d13c0 --- /dev/null +++ b/apps/internal-tool/spacetimedb/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/apps/internal-tool/src/app/app-client.tsx b/apps/internal-tool/src/app/app-client.tsx new file mode 100644 index 0000000000..6288ba3567 --- /dev/null +++ b/apps/internal-tool/src/app/app-client.tsx @@ -0,0 +1,203 @@ +import { useUser } from "@stackframe/react"; +import { useState } from "react"; +import { clsx } from "clsx"; +import { CallLogList } from "../components/CallLogList"; +import { CallLogDetail } from "../components/CallLogDetail"; +import { AddManualQa } from "../components/AddManualQa"; +import { KnowledgeBase } from "../components/KnowledgeBase"; +import { Analytics } from "../components/Analytics"; +import { useMcpCallLogs } from "../hooks/useSpacetimeDB"; +import type { McpCallLogRow } from "../types"; + +type Tab = "calls" | "knowledge" | "analytics"; + +export default function App() { + const user = useUser(); + const [selectedRow, setSelectedRow] = useState(null); + const [showAddQa, setShowAddQa] = useState(false); + const [tab, setTab] = useState("calls"); + const { rows, connectionState, connection } = useMcpCallLogs(); + + if (!user) { + return ( +
+
+

MCP Review Tool

+

+ Sign in to the{" "} + + Stack Dashboard + + {" "}first, then reload this page. +

+ +
+
+ ); + } + + if (process.env.NODE_ENV !== "development") { + const metadata = user.clientReadOnlyMetadata as Record | null; + if (!metadata?.approved) { + return ( +
+
+

Access Denied

+

+ You are signed in as {user.displayName ?? user.primaryEmail}, but your account is not approved. +

+

+ An admin needs to set approved: true in your client read-only metadata. +

+
+
+ ); + } + } + + const currentSelectedRow = selectedRow + ? rows.find(r => r.id === selectedRow.id) ?? selectedRow + : null; + + const reviewedBy = user.displayName ?? user.primaryEmail ?? "unknown"; + + return ( +
+
+
+

MCP Review Tool

+ {/* Tabs */} +
+ + + +
+
+
+ + {user.displayName ?? user.primaryEmail} +
+
+ + {showAddQa && ( + setShowAddQa(false)} + onSave={(question, answer, publish) => { + if (!connection) return; + connection.reducers.addManualQa({ + question, + answer, + publish, + reviewedBy, + }).catch(() => {}); + }} + /> + )} + + {tab === "calls" && ( +
+
+ +
+ {currentSelectedRow && ( + + )} +
+ )} + + {tab === "knowledge" && ( +
+ { + if (!connection) return; + connection.reducers.updateHumanCorrection({ + correlationId, + correctedQuestion: question, + correctedAnswer: answer, + publish, + reviewedBy, + }).catch(() => {}); + }} + onDelete={(correlationId) => { + if (!connection) return; + connection.reducers.deleteQaEntry({ + correlationId, + }).catch(() => {}); + }} + /> +
+ )} + + {tab === "analytics" && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/internal-tool/src/app/globals.css b/apps/internal-tool/src/app/globals.css new file mode 100644 index 0000000000..7012e083ef --- /dev/null +++ b/apps/internal-tool/src/app/globals.css @@ -0,0 +1,10 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} diff --git a/apps/internal-tool/src/app/layout.tsx b/apps/internal-tool/src/app/layout.tsx new file mode 100644 index 0000000000..2abb02a3a1 --- /dev/null +++ b/apps/internal-tool/src/app/layout.tsx @@ -0,0 +1,26 @@ +"use client"; + +import { Suspense } from "react"; +import { StackProvider, StackTheme } from "@stackframe/react"; +import { stackClientApp } from "../stack"; +import Loading from "./loading"; +import "./globals.css"; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + + Stack Auth — MCP Review Tool + + + + + }> + {children} + + + + + + ); +} diff --git a/apps/internal-tool/src/app/loading.tsx b/apps/internal-tool/src/app/loading.tsx new file mode 100644 index 0000000000..ebfed0abd9 --- /dev/null +++ b/apps/internal-tool/src/app/loading.tsx @@ -0,0 +1,10 @@ +export default function Loading() { + return ( +
+
+
+

Loading...

+
+
+ ); +} diff --git a/apps/internal-tool/src/app/page.tsx b/apps/internal-tool/src/app/page.tsx new file mode 100644 index 0000000000..7e07994b82 --- /dev/null +++ b/apps/internal-tool/src/app/page.tsx @@ -0,0 +1,7 @@ +"use client"; + +import App from "./app-client"; + +export default function Page() { + return ; +} diff --git a/apps/internal-tool/src/app/questions/page.tsx b/apps/internal-tool/src/app/questions/page.tsx new file mode 100644 index 0000000000..1ed1c5a7ea --- /dev/null +++ b/apps/internal-tool/src/app/questions/page.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { useMemo } from "react"; +import Markdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import { format } from "date-fns"; +import { useMcpCallLogs } from "../../hooks/useSpacetimeDB"; +import { toDate } from "../../utils"; +import { markdownComponents } from "../../components/markdown-components"; + +export default function QuestionsPage() { + const { rows, connectionState } = useMcpCallLogs(); + + const publishedQa = useMemo(() => { + return rows + .filter(r => r.publishedToQa) + .sort((a, b) => { + const aTime = a.publishedAt ? Number(toDate(a.publishedAt)) : 0; + const bTime = b.publishedAt ? Number(toDate(b.publishedAt)) : 0; + return bTime - aTime; + }); + }, [rows]); + + if (connectionState === "connecting") { + return ( +
+

Loading...

+
+ ); + } + + if (connectionState === "error") { + return ( +
+

Failed to connect to database.

+
+ ); + } + + return ( +
+

Stack Auth Q&A

+

+ Curated questions and answers about Stack Auth, reviewed by humans. +

+ + {publishedQa.length === 0 ? ( +

No published Q&A yet.

+ ) : ( +
+ {publishedQa.map(row => ( +
+

{row.humanCorrectedQuestion ?? row.question}

+
+ + {row.humanCorrectedAnswer ?? row.response} + +
+
+ {row.toolName} + {row.publishedAt && ( + {format(toDate(row.publishedAt), "MMM d, yyyy")} + )} + {row.humanReviewedBy && ( + Reviewed by {row.humanReviewedBy} + )} +
+
+ ))} +
+ )} +
+ ); +} diff --git a/apps/internal-tool/src/components/AddManualQa.tsx b/apps/internal-tool/src/components/AddManualQa.tsx new file mode 100644 index 0000000000..a58e42f37f --- /dev/null +++ b/apps/internal-tool/src/components/AddManualQa.tsx @@ -0,0 +1,101 @@ +import { useState } from "react"; +import { clsx } from "clsx"; + +export function AddManualQa({ onClose, onSave }: { + onClose: () => void; + onSave: (question: string, answer: string, publish: boolean) => void; +}) { + const [question, setQuestion] = useState(""); + const [answer, setAnswer] = useState(""); + const [saved, setSaved] = useState(false); + + const canSave = question.trim().length > 0 && answer.trim().length > 0; + + const handleSave = (publish: boolean) => { + if (!canSave) return; + onSave(question.trim(), answer.trim(), publish); + setSaved(true); + setTimeout(() => { + setSaved(false); + setQuestion(""); + setAnswer(""); + if (publish) { + onClose(); + } + }, 1500); + }; + + return ( +
+
+ {/* Header */} +
+

Add Q&A

+ +
+ + {/* Form */} +
+ {saved && ( +
+ Saved successfully +
+ )} + +
+ + setQuestion(e.target.value)} + placeholder="e.g. How do I set up OAuth with Stack Auth?" + /> +
+ +
+ +