diff --git a/.github/workflows/db-migration-backwards-compatibility.yaml b/.github/workflows/db-migration-backwards-compatibility.yaml index 122e5d6638..829abf7ae3 100644 --- a/.github/workflows/db-migration-backwards-compatibility.yaml +++ b/.github/workflows/db-migration-backwards-compatibility.yaml @@ -138,6 +138,9 @@ jobs: - name: Create .env.test.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.test.local + - name: Create .env.test.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local + - name: Build run: pnpm build @@ -332,6 +335,9 @@ jobs: - name: Create .env.test.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.test.local + - name: Create .env.test.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local + - name: Build run: pnpm build diff --git a/.github/workflows/e2e-api-tests-local-emulator.yaml b/.github/workflows/e2e-api-tests-local-emulator.yaml index 0b1ab5e433..3ebed27862 100644 --- a/.github/workflows/e2e-api-tests-local-emulator.yaml +++ b/.github/workflows/e2e-api-tests-local-emulator.yaml @@ -86,6 +86,9 @@ jobs: - name: Create .env.test.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.test.local + - name: Create .env.test.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local + - name: Build run: pnpm build diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml index d949d129ac..d3d58d132c 100644 --- a/.github/workflows/e2e-api-tests.yaml +++ b/.github/workflows/e2e-api-tests.yaml @@ -92,6 +92,9 @@ jobs: - name: Create .env.test.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.test.local + - name: Create .env.test.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local + - name: Build run: pnpm build diff --git a/.github/workflows/e2e-custom-base-port-api-tests.yaml b/.github/workflows/e2e-custom-base-port-api-tests.yaml index dc0a7f4c85..95b7f680d3 100644 --- a/.github/workflows/e2e-custom-base-port-api-tests.yaml +++ b/.github/workflows/e2e-custom-base-port-api-tests.yaml @@ -85,6 +85,9 @@ jobs: - name: Create .env.test.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.test.local + - name: Create .env.test.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local + - name: Build run: pnpm build diff --git a/.github/workflows/e2e-fallback-tests.yaml b/.github/workflows/e2e-fallback-tests.yaml index d5b0a5fdca..93d51e5f43 100644 --- a/.github/workflows/e2e-fallback-tests.yaml +++ b/.github/workflows/e2e-fallback-tests.yaml @@ -68,6 +68,7 @@ jobs: cp examples/middleware/.env.development examples/middleware/.env.test.local cp examples/supabase/.env.development examples/supabase/.env.test.local cp examples/convex/.env.development examples/convex/.env.test.local + cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local - name: Build run: pnpm build diff --git a/.github/workflows/lint-and-build.yaml b/.github/workflows/lint-and-build.yaml index 7b70deff4b..fe32a5e2f3 100644 --- a/.github/workflows/lint-and-build.yaml +++ b/.github/workflows/lint-and-build.yaml @@ -66,6 +66,9 @@ jobs: - name: Create .env.production.local file for examples/convex run: cp examples/convex/.env.development examples/convex/.env.production.local + - name: Create .env.production.local file for apps/internal-tool + run: cp apps/internal-tool/.env.development apps/internal-tool/.env.production.local + - name: Build run: pnpm build diff --git a/README.md b/README.md index 33388ae06b..705993615a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ [![Stack Logo](/.github/assets/logo.png)](https://stack-auth.com) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/stack-auth/stack-auth) +

📘 Docs | ☁️ Hosted Version diff --git a/apps/backend/.env b/apps/backend/.env index fc0277d588..6f28c1c4ef 100644 --- a/apps/backend/.env +++ b/apps/backend/.env @@ -118,3 +118,8 @@ STACK_TELEGRAM_CHAT_ID=# enter your telegram chat id # Docs AI tool bundle STACK_DOCS_INTERNAL_BASE_URL=# override the docs origin used by the backend's AI tool bundle to call the docs app's `/api/internal/docs-tools` endpoint. Defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod + +# MCP review tool (SpacetimeDB) +STACK_SPACETIMEDB_URI=# SpacetimeDB host URI; default empty (logging disabled) +STACK_SPACETIMEDB_DB_NAME=# SpacetimeDB database name +STACK_MCP_LOG_TOKEN=# shared secret gating the log_mcp_call reducer; must match EXPECTED_LOG_TOKEN in apps/internal-tool/spacetimedb/src/index.ts diff --git a/apps/backend/.env.development b/apps/backend/.env.development index ea0695a6d0..8ec5f41785 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -113,6 +113,11 @@ STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs +# MCP review tool (SpacetimeDB) +STACK_SPACETIMEDB_URI=ws://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39 +STACK_SPACETIMEDB_DB_NAME=stack-auth-llm +STACK_MCP_LOG_TOKEN=change-me + # Clickhouse STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36 STACK_CLICKHOUSE_ADMIN_USER=stackframe diff --git a/apps/backend/package.json b/apps/backend/package.json index b1f80a58b2..d8198c74f8 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -55,6 +55,7 @@ }, "dependencies": { "@ai-sdk/mcp": "^1.0.21", + "spacetimedb": "^2.1.0", "@ai-sdk/openai": "^3.0.29", "@aws-sdk/client-s3": "^3.855.0", "@clickhouse/client": "^1.14.0", diff --git a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts index 82f5fbbe00..959a859d98 100644 --- a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts +++ b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts @@ -1,10 +1,14 @@ +import { logMcpCall } from "@/lib/ai/mcp-logger"; import { selectModel } from "@/lib/ai/models"; import { getFullSystemPrompt } from "@/lib/ai/prompts"; +import { reviewMcpCall } from "@/lib/ai/qa-reviewer"; import { requestBodySchema } from "@/lib/ai/schema"; import { getTools, validateToolNames } from "@/lib/ai/tools"; +import { getVerifiedQaContext } from "@/lib/ai/verified-qa"; import { listManagedProjectIds } from "@/lib/projects"; import { SmartResponse } from "@/route-handlers/smart-response"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; import { validateImageAttachments } from "@stackframe/stack-shared/dist/ai/image-limits"; import { yupMixed, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; @@ -52,15 +56,14 @@ export const POST = createSmartRouteHandler({ } const model = selectModel(quality, speed, isAuthenticated); - const systemPrompt = getFullSystemPrompt(systemPromptId); + const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; + let systemPrompt = getFullSystemPrompt(systemPromptId); + if (isDocsOrSearch) { + systemPrompt += await getVerifiedQaContext(); + } const tools = await getTools(toolNames, { auth: fullReq.auth, targetProjectId: projectId }); const toolsArg = Object.keys(tools).length > 0 ? tools : undefined; - const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; - // create-dashboard now does an inspection loop (queryAnalytics) before calling updateDashboard, - // so it needs room for ~3 exploratory queries + the final tool call + some retry slack. const isCreateDashboard = systemPromptId === "create-dashboard"; - // build-analytics-query aims for one-shot queries with complete schema - // knowledge, but needs a few steps for retries on errors or follow-ups. const isBuildAnalyticsQuery = systemPromptId === "build-analytics-query"; const stepLimit = toolsArg == null ? 1 @@ -86,6 +89,7 @@ export const POST = createSmartRouteHandler({ body: result.toUIMessageStreamResponse(), }; } else { + const startedAt = Date.now(); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 120_000); const result = await generateText({ @@ -134,10 +138,51 @@ export const POST = createSmartRouteHandler({ }); }); + let responseConversationId: string | undefined; + if (body.mcpCallMetadata != null) { + const correlationId = crypto.randomUUID(); + const conversationId = body.mcpCallMetadata.conversationId ?? crypto.randomUUID(); + responseConversationId = conversationId; + const firstUserMessage = messages.find(m => m.role === "user"); + const question = typeof firstUserMessage?.content === "string" + ? firstUserMessage.content + : JSON.stringify(firstUserMessage?.content ?? ""); + + const innerToolCallsJson = JSON.stringify(contentBlocks.filter(b => b.type === "tool-call")); + + const logPromise = logMcpCall({ + correlationId, + toolName: body.mcpCallMetadata.toolName, + reason: body.mcpCallMetadata.reason, + userPrompt: body.mcpCallMetadata.userPrompt, + conversationId, + question, + response: result.text, + stepCount: result.steps.length, + innerToolCallsJson, + durationMs: BigInt(Date.now() - startedAt), + modelId: String(model.modelId), + errorMessage: undefined, + }); + runAsynchronouslyAndWaitUntil(logPromise); + + runAsynchronouslyAndWaitUntil(reviewMcpCall({ + logPromise, + correlationId, + question, + reason: body.mcpCallMetadata.reason, + response: result.text, + })); + } + return { statusCode: 200, bodyType: "json" as const, - body: { content: contentBlocks, finalText: result.text }, + body: { + content: contentBlocks, + finalText: result.text, + conversationId: responseConversationId ?? null, + }, }; } }, diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts new file mode 100644 index 0000000000..fed36bcc9f --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts @@ -0,0 +1,55 @@ +import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + question: yupString().defined(), + answer: yupString().defined(), + publish: yupBoolean().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + const user = auth.user; + if (getNodeEnvironment() !== "development") { + const metadata = user.client_read_only_metadata; + if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { + throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); + } + } + + const conn = await getConnectionOrThrow(); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.addManualQa({ + token, + question: body.question, + answer: body.answer, + publish: body.publish, + reviewedBy: user.display_name ?? user.primary_email ?? user.id, + }); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts new file mode 100644 index 0000000000..e3126c65d0 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts @@ -0,0 +1,49 @@ +import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + correlationId: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + if (getNodeEnvironment() !== "development") { + const metadata = auth.user.client_read_only_metadata; + if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { + throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); + } + } + + const conn = await getConnectionOrThrow(); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.deleteQaEntry({ + token, + correlationId: body.correlationId, + }); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts new file mode 100644 index 0000000000..5500fe15ba --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts @@ -0,0 +1,51 @@ +import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + correlationId: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + const user = auth.user; + if (getNodeEnvironment() !== "development") { + const metadata = user.client_read_only_metadata; + if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { + throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); + } + } + + const conn = await getConnectionOrThrow(); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.markHumanReviewed({ + token, + correlationId: body.correlationId, + reviewedBy: user.display_name ?? user.primary_email ?? user.id, + }); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts new file mode 100644 index 0000000000..dfb40a7859 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts @@ -0,0 +1,57 @@ +import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + correlationId: yupString().defined(), + correctedQuestion: yupString().defined(), + correctedAnswer: yupString().defined(), + publish: yupBoolean().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + const user = auth.user; + if (getNodeEnvironment() !== "development") { + const metadata = user.client_read_only_metadata; + if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { + throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); + } + } + + const conn = await getConnectionOrThrow(); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.updateHumanCorrection({ + token, + correlationId: body.correlationId, + correctedQuestion: body.correctedQuestion, + correctedAnswer: body.correctedAnswer, + publish: body.publish, + reviewedBy: user.display_name ?? user.primary_email ?? user.id, + }); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/lib/ai/mcp-logger.ts b/apps/backend/src/lib/ai/mcp-logger.ts new file mode 100644 index 0000000000..f8a9a3d045 --- /dev/null +++ b/apps/backend/src/lib/ai/mcp-logger.ts @@ -0,0 +1,59 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError, captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { DbConnection } from "./spacetimedb-bindings"; +import type { LogMcpCallParams } from "./spacetimedb-bindings/types/reducers"; + +export type McpLogEntry = Omit; + +let connectionPromise: Promise | null = null; + +export async function getConnection(): Promise { + const uri = getEnvVariable("STACK_SPACETIMEDB_URI", ""); + if (!uri) { + return null; + } + + if (!connectionPromise) { + connectionPromise = new Promise((resolve, reject) => { + DbConnection.builder() + .withUri(uri) + .withDatabaseName(getEnvVariable("STACK_SPACETIMEDB_DB_NAME")) + .onConnect((connInstance) => { + connInstance.subscriptionBuilder() + .onApplied(() => { + resolve(connInstance); + }) + .subscribe("SELECT * FROM mcp_call_log"); + }) + .onConnectError((_: unknown, err: Error) => { + captureError("mcp-logger", err); + connectionPromise = null; + reject(err); + }) + .build(); + }); + } + + return await connectionPromise; +} + +export async function getConnectionOrThrow(): Promise { + const conn = await getConnection(); + if (!conn) { + throw new StackAssertionError("SpacetimeDB connection unavailable"); + } + return conn; +} + +export async function logMcpCall(entry: McpLogEntry): Promise { + const conn = await getConnection(); + if (!conn) { + return; + } + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.logMcpCall({ + token, + ...entry, + }); +} diff --git a/apps/backend/src/lib/ai/prompts.ts b/apps/backend/src/lib/ai/prompts.ts index 7a66a4d3d7..7bf5eae96a 100644 --- a/apps/backend/src/lib/ai/prompts.ts +++ b/apps/backend/src/lib/ai/prompts.ts @@ -161,11 +161,12 @@ You are Stack Auth's AI assistant. You help users with Stack Auth - a complete a Think step by step about what to say. Being wrong is 100x worse than saying you don't know. -## TOOL USAGE WORKFLOW: -1. **FIRST**, use \`search_docs\` with relevant keywords to find related documentation -2. **THEN**, use \`get_docs_by_id\` to retrieve the full content of the most relevant pages -3. Base your answer on the actual documentation content retrieved -4. When referring to API endpoints, **always cite the actual endpoint** (e.g., "GET /users/me") not the documentation URL +## PRIORITY ORDER: +1. **FIRST**, check the Human-Verified Knowledge Base (appended at the end of this prompt, if any). If the user's question is an exact or near-exact match to a verified Q&A, you may use that answer verbatim without searching docs. +2. **OTHERWISE**, use \`search_docs\` with relevant keywords to find related documentation — this is mandatory when there is no exact verified-QA match. +3. **THEN**, use \`get_docs_by_id\` to retrieve the full content of the most relevant pages +4. Base your answer on the actual documentation content retrieved +5. When referring to API endpoints, **always cite the actual endpoint** (e.g., "GET /users/me") not the documentation URL ## CORE RESPONSIBILITIES: 1. Help users implement Stack Auth in their applications diff --git a/apps/backend/src/lib/ai/qa-reviewer.ts b/apps/backend/src/lib/ai/qa-reviewer.ts new file mode 100644 index 0000000000..16d0a97de6 --- /dev/null +++ b/apps/backend/src/lib/ai/qa-reviewer.ts @@ -0,0 +1,184 @@ +import { createMCPClient } from "@ai-sdk/mcp"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { generateText, stepCountIs } from "ai"; +import { getConnection } from "./mcp-logger"; +import { createOpenRouterProvider } from "./models"; +import { getVerifiedQaContext } from "./verified-qa"; + +const QA_SYSTEM_PROMPT = `You are a QA reviewer for Stack Auth's AI documentation assistant. +You will receive a question, the agent's stated reason for asking, and the AI's response. + +Your tasks: +1. RELEVANCE: Does the response actually answer the question? Does the stated reason align with what was asked? +2. CORRECTNESS: Verify factual claims about Stack Auth. Use human-verified Q&A (appended below, if any) as the highest-priority source of truth — these are always correct. Then use the available tools to look up additional information from the Stack Auth codebase. If the AI response contradicts a human-verified answer, flag it as incorrect. + +The repo name for all tool calls is "stack-auth/stack-auth". Only use the repository documentation tools (read_wiki_structure, read_wiki_contents, ask_question) — do not create sessions or modify any other resources. + +You MUST respond with ONLY valid JSON matching this exact schema (no markdown, no explanation outside the JSON): +{ + "needsHumanReview": boolean, + "answerCorrect": boolean, + "answerRelevant": boolean, + "flags": [{"type": string, "severity": "low" | "medium" | "high" | "critical", "explanation": string}], + "improvementSuggestions": string, + "overallScore": number +} + +Flag types: "factual_error", "incomplete_answer", "off_topic", "hallucination", "outdated_info", "missing_context", "misleading", "reason_mismatch" + +Scoring: +- 90-100: Excellent — factually correct, fully addresses the question +- 70-89: Good — minor issues or missing details +- 50-69: Acceptable — notable issues but core answer is present +- 30-49: Poor — significant problems +- 0-29: Unacceptable — fundamentally wrong or irrelevant + +Set needsHumanReview=true if: score < 50, any critical flag, or you are uncertain about correctness.`; + +const REVIEW_MODEL_ID = "anthropic/claude-haiku-4.5"; + +export async function reviewMcpCall(entry: { + logPromise: Promise; + correlationId: string; + question: string; + reason: string; + response: string; +}): Promise { + const apiKey = getEnvVariable("STACK_OPENROUTER_API_KEY", ""); + if (!apiKey || apiKey === "FORWARD_TO_PRODUCTION") { + return; + } + + let devinClient: Awaited> | null = null; + + const failureUpdate = (err: unknown) => ({ + qaNeedsHumanReview: true, + qaAnswerCorrect: false, + qaAnswerRelevant: false, + qaFlagsJson: "[]", + qaImprovementSuggestions: "", + qaOverallScore: 0, + qaConversationJson: undefined, + qaErrorMessage: String(err), + }); + + let update: { + qaNeedsHumanReview: boolean, + qaAnswerCorrect: boolean, + qaAnswerRelevant: boolean, + qaFlagsJson: string, + qaImprovementSuggestions: string, + qaOverallScore: number, + qaConversationJson: string | undefined, + qaErrorMessage: string | undefined, + }; + + try { + // Wait for the log row to be written first + await entry.logPromise; + + devinClient = await createMCPClient({ + transport: { + type: "http", + url: "https://mcp.deepwiki.com/mcp", + }, + }); + + const devinTools = await devinClient.tools(); + const openrouter = createOpenRouterProvider(); + const model = openrouter(REVIEW_MODEL_ID); + + const userMessage = [ + "## Question", + entry.question, + "", + "## Agent's Reason for Asking", + entry.reason, + "", + "## AI Response", + entry.response, + ].join("\n"); + + const verifiedQa = await getVerifiedQaContext(); + + const result = await generateText({ + model, + system: QA_SYSTEM_PROMPT + verifiedQa, + tools: devinTools as Parameters[0]["tools"], + stopWhen: stepCountIs(10), + messages: [{ role: "user", content: userMessage }], + }); + + const conversation = result.steps.map((step, i) => { + const toolCalls = step.toolCalls.map(tc => ({ toolName: tc.toolName, toolCallId: tc.toolCallId, args: tc.input })); + const toolResults = step.toolResults.map(tr => ({ + toolName: tr.toolName, + toolCallId: tr.toolCallId, + result: tr.output, + })); + return { + step: i + 1, + text: step.text || undefined, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + }; + }); + + const jsonMatch = result.text.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error("No JSON found in QA review response"); + } + const raw = JSON.parse(jsonMatch[0]); + if ( + typeof raw.needsHumanReview !== "boolean" || + typeof raw.answerCorrect !== "boolean" || + typeof raw.answerRelevant !== "boolean" || + !Array.isArray(raw.flags) || + typeof raw.overallScore !== "number" + ) { + throw new Error(`Invalid QA review response shape: ${JSON.stringify(raw).slice(0, 200)}`); + } + const parsed = raw as { + needsHumanReview: boolean, + answerCorrect: boolean, + answerRelevant: boolean, + flags: Array<{ type: string, severity: string, explanation: string }>, + improvementSuggestions: string, + overallScore: number, + }; + parsed.overallScore = Math.max(0, Math.min(100, Math.round(parsed.overallScore))); + + update = { + qaNeedsHumanReview: parsed.needsHumanReview, + qaAnswerCorrect: parsed.answerCorrect, + qaAnswerRelevant: parsed.answerRelevant, + qaFlagsJson: JSON.stringify(parsed.flags), + qaImprovementSuggestions: parsed.improvementSuggestions, + qaOverallScore: parsed.overallScore, + qaConversationJson: JSON.stringify(conversation), + qaErrorMessage: undefined, + }; + } catch (err) { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + update = failureUpdate(err); + } + + if (devinClient) { + await devinClient.close().catch((err: unknown) => { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + }); + } + + const conn = await getConnection(); + if (!conn) return; + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await conn.reducers.updateMcpQaReview({ + token, + correlationId: entry.correlationId, + qaReviewModelId: REVIEW_MODEL_ID, + ...update, + }).catch((err: unknown) => { + captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + }); +} diff --git a/apps/backend/src/lib/ai/schema.ts b/apps/backend/src/lib/ai/schema.ts index 3da71d7b21..12a44d96b6 100644 --- a/apps/backend/src/lib/ai/schema.ts +++ b/apps/backend/src/lib/ai/schema.ts @@ -25,6 +25,12 @@ export const requestBodySchema = yupObject({ }).defined() ).defined().min(1), projectId: yupString().optional().nullable(), + mcpCallMetadata: yupObject({ + toolName: yupString().defined(), + reason: yupString().defined(), + userPrompt: yupString().defined(), + conversationId: yupString().optional().nullable(), + }).optional().nullable(), }); export type RequestBody = InferType; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts new file mode 100644 index 0000000000..97d707b5c7 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + question: __t.string(), + answer: __t.string(), + publish: __t.bool(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts new file mode 100644 index 0000000000..1fa706ca53 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts new file mode 100644 index 0000000000..bf16c63e07 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts @@ -0,0 +1,129 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb cli version 2.1.0 (commit 6981f48b4bc1a71c8dd9bdfe5a2c343f6370243d). + +/* eslint-disable */ +/* tslint:disable */ +import { + DbConnectionBuilder as __DbConnectionBuilder, + DbConnectionImpl as __DbConnectionImpl, + SubscriptionBuilderImpl as __SubscriptionBuilderImpl, + TypeBuilder as __TypeBuilder, + Uuid as __Uuid, + convertToAccessorMap as __convertToAccessorMap, + makeQueryBuilder as __makeQueryBuilder, + procedureSchema as __procedureSchema, + procedures as __procedures, + reducerSchema as __reducerSchema, + reducers as __reducers, + schema as __schema, + t as __t, + table as __table, + type AlgebraicTypeType as __AlgebraicTypeType, + type DbConnectionConfig as __DbConnectionConfig, + type ErrorContextInterface as __ErrorContextInterface, + type Event as __Event, + type EventContextInterface as __EventContextInterface, + type Infer as __Infer, + type QueryBuilder as __QueryBuilder, + type ReducerEventContextInterface as __ReducerEventContextInterface, + type RemoteModule as __RemoteModule, + type SubscriptionEventContextInterface as __SubscriptionEventContextInterface, + type SubscriptionHandleImpl as __SubscriptionHandleImpl, +} from "spacetimedb"; + +// Import all reducer arg schemas +import AddManualQaReducer from "./add_manual_qa_reducer"; +import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; +import LogMcpCallReducer from "./log_mcp_call_reducer"; +import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; +import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; +import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; + +// Import all procedure arg schemas + +// Import all table schema definitions +import McpCallLogRow from "./mcp_call_log_table"; + +/** Type-only namespace exports for generated type groups. */ + +/** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ +const tablesSchema = __schema({ + mcpCallLog: __table({ + name: 'mcp_call_log', + indexes: [ + { accessor: 'id', name: 'mcp_call_log_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'mcp_call_log_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, McpCallLogRow), +}); + +/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ +const reducersSchema = __reducers( + __reducerSchema("add_manual_qa", AddManualQaReducer), + __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), + __reducerSchema("log_mcp_call", LogMcpCallReducer), + __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), + __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), + __reducerSchema("update_mcp_qa_review", UpdateMcpQaReviewReducer), +); + +/** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ +const proceduresSchema = __procedures( +); + +/** The remote SpacetimeDB module schema, both runtime and type information. */ +const REMOTE_MODULE = { + versionInfo: { + cliVersion: "2.1.0" as const, + }, + tables: tablesSchema.schemaType.tables, + reducers: reducersSchema.reducersType.reducers, + ...proceduresSchema, +} satisfies __RemoteModule< + typeof tablesSchema.schemaType, + typeof reducersSchema.reducersType, + typeof proceduresSchema +>; + +/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */ +export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType); + +/** The reducers available in this remote SpacetimeDB module. */ +export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers); + +/** The context type returned in callbacks for all possible events. */ +export type EventContext = __EventContextInterface; +/** The context type returned in callbacks for reducer events. */ +export type ReducerEventContext = __ReducerEventContextInterface; +/** The context type returned in callbacks for subscription events. */ +export type SubscriptionEventContext = __SubscriptionEventContextInterface; +/** The context type returned in callbacks for error events. */ +export type ErrorContext = __ErrorContextInterface; +/** The subscription handle type to manage active subscriptions created from a {@link SubscriptionBuilder}. */ +export type SubscriptionHandle = __SubscriptionHandleImpl; + +/** Builder class to configure a new subscription to the remote SpacetimeDB instance. */ +export class SubscriptionBuilder extends __SubscriptionBuilderImpl {} + +/** Builder class to configure a new database connection to the remote SpacetimeDB instance. */ +export class DbConnectionBuilder extends __DbConnectionBuilder {} + +/** The typed database connection to manage connections to the remote SpacetimeDB instance. This class has type information specific to the generated module. */ +export class DbConnection extends __DbConnectionImpl { + /** Creates a new {@link DbConnectionBuilder} to configure and connect to the remote SpacetimeDB instance. */ + static builder = (): DbConnectionBuilder => { + return new DbConnectionBuilder(REMOTE_MODULE, (config: __DbConnectionConfig) => new DbConnection(config)); + }; + + /** Creates a new {@link SubscriptionBuilder} to configure a subscription to the remote SpacetimeDB instance. */ + override subscriptionBuilder = (): SubscriptionBuilder => { + return new SubscriptionBuilder(this); + }; +} + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts new file mode 100644 index 0000000000..772cd1fc78 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts @@ -0,0 +1,27 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + conversationId: __t.option(__t.string()), + toolName: __t.string(), + reason: __t.string(), + userPrompt: __t.string(), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32(), + innerToolCallsJson: __t.string(), + durationMs: __t.u64(), + modelId: __t.string(), + errorMessage: __t.option(__t.string()), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts new file mode 100644 index 0000000000..87a1e7d923 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts new file mode 100644 index 0000000000..bcf0dab879 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts @@ -0,0 +1,44 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + correlationId: __t.string().name("correlation_id"), + conversationId: __t.option(__t.string()).name("conversation_id"), + createdAt: __t.timestamp().name("created_at"), + toolName: __t.string().name("tool_name"), + reason: __t.string(), + userPrompt: __t.string().name("user_prompt"), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32().name("step_count"), + innerToolCallsJson: __t.string().name("inner_tool_calls_json"), + durationMs: __t.u64().name("duration_ms"), + modelId: __t.string().name("model_id"), + errorMessage: __t.option(__t.string()).name("error_message"), + qaReviewedAt: __t.option(__t.timestamp()).name("qa_reviewed_at"), + qaNeedsHumanReview: __t.option(__t.bool()).name("qa_needs_human_review"), + qaAnswerCorrect: __t.option(__t.bool()).name("qa_answer_correct"), + qaAnswerRelevant: __t.option(__t.bool()).name("qa_answer_relevant"), + qaFlagsJson: __t.option(__t.string()).name("qa_flags_json"), + qaImprovementSuggestions: __t.option(__t.string()).name("qa_improvement_suggestions"), + qaOverallScore: __t.option(__t.u32()).name("qa_overall_score"), + qaReviewModelId: __t.option(__t.string()).name("qa_review_model_id"), + qaConversationJson: __t.option(__t.string()).name("qa_conversation_json"), + qaErrorMessage: __t.option(__t.string()).name("qa_error_message"), + humanReviewedAt: __t.option(__t.timestamp()).name("human_reviewed_at"), + humanReviewedBy: __t.option(__t.string()).name("human_reviewed_by"), + humanCorrectedQuestion: __t.option(__t.string()).name("human_corrected_question"), + humanCorrectedAnswer: __t.option(__t.string()).name("human_corrected_answer"), + publishedToQa: __t.option(__t.bool()).name("published_to_qa"), + publishedAt: __t.option(__t.timestamp()).name("published_at"), +}); diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts new file mode 100644 index 0000000000..4af9e7b736 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts @@ -0,0 +1,46 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const McpCallLog = __t.object("McpCallLog", { + id: __t.u64(), + correlationId: __t.string(), + conversationId: __t.option(__t.string()), + createdAt: __t.timestamp(), + toolName: __t.string(), + reason: __t.string(), + userPrompt: __t.string(), + question: __t.string(), + response: __t.string(), + stepCount: __t.u32(), + innerToolCallsJson: __t.string(), + durationMs: __t.u64(), + modelId: __t.string(), + errorMessage: __t.option(__t.string()), + qaReviewedAt: __t.option(__t.timestamp()), + qaNeedsHumanReview: __t.option(__t.bool()), + qaAnswerCorrect: __t.option(__t.bool()), + qaAnswerRelevant: __t.option(__t.bool()), + qaFlagsJson: __t.option(__t.string()), + qaImprovementSuggestions: __t.option(__t.string()), + qaOverallScore: __t.option(__t.u32()), + qaReviewModelId: __t.option(__t.string()), + qaConversationJson: __t.option(__t.string()), + qaErrorMessage: __t.option(__t.string()), + humanReviewedAt: __t.option(__t.timestamp()), + humanReviewedBy: __t.option(__t.string()), + humanCorrectedQuestion: __t.option(__t.string()), + humanCorrectedAnswer: __t.option(__t.string()), + publishedToQa: __t.option(__t.bool()), + publishedAt: __t.option(__t.timestamp()), +}); +export type McpCallLog = __Infer; + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts new file mode 100644 index 0000000000..d5ac825c9a --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts @@ -0,0 +1,10 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all procedure arg schemas + + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts new file mode 100644 index 0000000000..87a6606ae4 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts @@ -0,0 +1,22 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all reducer arg schemas +import AddManualQaReducer from "../add_manual_qa_reducer"; +import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; +import LogMcpCallReducer from "../log_mcp_call_reducer"; +import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; +import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; +import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; + +export type AddManualQaParams = __Infer; +export type DeleteQaEntryParams = __Infer; +export type LogMcpCallParams = __Infer; +export type MarkHumanReviewedParams = __Infer; +export type UpdateHumanCorrectionParams = __Infer; +export type UpdateMcpQaReviewParams = __Infer; + diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts new file mode 100644 index 0000000000..2ae110a192 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + correctedQuestion: __t.string(), + correctedAnswer: __t.string(), + publish: __t.bool(), + reviewedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts new file mode 100644 index 0000000000..035e1d017c --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts @@ -0,0 +1,25 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + qaNeedsHumanReview: __t.bool(), + qaAnswerCorrect: __t.bool(), + qaAnswerRelevant: __t.bool(), + qaFlagsJson: __t.string(), + qaImprovementSuggestions: __t.string(), + qaOverallScore: __t.u32(), + qaReviewModelId: __t.string(), + qaConversationJson: __t.option(__t.string()), + qaErrorMessage: __t.option(__t.string()), +}; diff --git a/apps/backend/src/lib/ai/tools/docs.ts b/apps/backend/src/lib/ai/tools/docs.ts index f13fd5a971..f03f90f7ed 100644 --- a/apps/backend/src/lib/ai/tools/docs.ts +++ b/apps/backend/src/lib/ai/tools/docs.ts @@ -1,6 +1,6 @@ -import { tool } from "ai"; import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { tool } from "ai"; import { z } from "zod"; type DocsToolHttpResult = { @@ -15,41 +15,45 @@ function getDocsToolsBaseUrl(): string { } if (getNodeEnvironment() === "development") { const portPrefix = getEnvVariable("NEXT_PUBLIC_STACK_PORT_PREFIX", "81"); - return `http://localhost:${portPrefix}04`; + return `http://localhost:${portPrefix}26`; } return "https://mcp.stack-auth.com"; } async function postDocsToolAction(action: Record): Promise { const base = getDocsToolsBaseUrl(); + try { + const res = await fetch(`${base}/api/internal/docs-tools`, { + method: "POST", + headers: { + "Content-Type": "application/json", + // MCP-style JSON-RPC endpoint requires clients to advertise both JSON and SSE. + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify(action), + }); - const res = await fetch(`${base}/api/internal/docs-tools`, { - method: "POST", - headers: { - "Content-Type": "application/json", - // MCP-style JSON-RPC endpoint requires clients to advertise both JSON and SSE. - Accept: "application/json, text/event-stream", - }, - body: JSON.stringify(action), - }); + if (!res.ok) { + const errBody = await res.text(); + captureError("docs-tools-http-error", new Error(`Stack Auth docs tools error (${res.status}): ${errBody}`)); + return "Stack Auth docs tools returned an error. Please try again later."; + } - if (!res.ok) { - const errBody = await res.text(); - captureError("docs-tools-http-error", new Error(`Stack Auth docs tools error (${res.status}): ${errBody}`)); - return `Stack Auth docs tools error (${res.status}): ${errBody}`; - } + const data = (await res.json()) as DocsToolHttpResult; + const text = data.content + ?.filter((c): c is { type: "text", text: string } => c.type === "text" && typeof c.text === "string") + .map((c) => c.text) + .join("\n") ?? ""; - const data = (await res.json()) as DocsToolHttpResult; - const text = data.content - ?.filter((c): c is { type: "text", text: string } => c.type === "text" && typeof c.text === "string") - .map((c) => c.text) - .join("\n") ?? ""; + if (data.isError === true) { + return text || "Unknown docs tool error"; + } - if (data.isError === true) { - return text || "Unknown docs tool error"; + return text; + } catch (err) { + captureError("docs-tools-transport-error", err instanceof Error ? err : new Error(String(err))); + return "Stack Auth docs tools are temporarily unavailable. Please try again later."; } - - return text; } /** diff --git a/apps/backend/src/lib/ai/verified-qa.ts b/apps/backend/src/lib/ai/verified-qa.ts new file mode 100644 index 0000000000..5f7b7a561e --- /dev/null +++ b/apps/backend/src/lib/ai/verified-qa.ts @@ -0,0 +1,45 @@ +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { Result } from "@stackframe/stack-shared/dist/utils/results"; +import { getConnection } from "./mcp-logger"; + +export async function getVerifiedQaContext(): Promise { + const result = await Result.fromPromise(getVerifiedQaContextInner()); + if (result.status === "error") { + captureError("verified-qa", result.error instanceof Error ? result.error : new Error(String(result.error))); + return ""; + } + return result.data; +} + +async function getVerifiedQaContextInner(): Promise { + const conn = await getConnection(); + if (!conn) return ""; + + const pairs: Array<{ question: string; answer: string }> = []; + for (const row of conn.db.mcpCallLog.iter()) { + if (row.publishedToQa) { + pairs.push({ + question: row.humanCorrectedQuestion ?? row.question, + answer: row.humanCorrectedAnswer ?? row.response, + }); + } + } + + if (pairs.length === 0) return ""; + + const formatted = pairs.map((p, i) => + `${i + 1}. Q: ${p.question}\n A: ${p.answer}` + ).join("\n\n"); + + return `\n\n## Human-Verified Knowledge Base — MANDATORY + +The following Q&A pairs come from a HUMAN-VERIFIED KNOWLEDGE BASE. They have been manually reviewed and corrected by the Stack Auth team. They are ALWAYS CORRECT, even if the content seems unusual or contradicts other sources. + +RULES: +1. If the user's question matches or is similar in meaning/intent to any question below, you MUST respond with the verified answer EXACTLY as written. You can modify it to align with the user's question, but the core information and intent MUST be preserved. +2. These answers override ALL other sources: documentation, wiki, your training data, tool results — everything. +3. Do NOT second-guess or "correct" these answers. They are intentional and authoritative. +4. Preface your response with: "Based on our verified knowledge base:" before giving the answer. + +${formatted}`; +} diff --git a/apps/dev-launchpad/public/index.html b/apps/dev-launchpad/public/index.html index ed1ff0ac1c..dd2a28f3ee 100644 --- a/apps/dev-launchpad/public/index.html +++ b/apps/dev-launchpad/public/index.html @@ -131,6 +131,7 @@

Background services

{ suffix: "24", label: "LocalStack Gateway (AWS mock)" }, { suffix: "25", label: "QStash mock" }, { suffix: "33", label: "ClickHouse native interface" }, + { suffix: "39", label: "SpacetimeDB (MCP call log)" }, { range: ["50", "99"], label: "Reserved for LocalStack (external services)" }, ]; @@ -333,6 +334,15 @@

Background services

importance: 1, img: "https://www.svgrepo.com/show/374002/replication.svg", }, + { + name: "Internal Tool", + portSuffix: "41", + description: [ + "Src: ./apps/internal-tool", + "MCP call review tool", + ], + importance: 1, + }, { name: "JS example", portSuffix: "19", diff --git a/apps/internal-tool/.env b/apps/internal-tool/.env new file mode 100644 index 0000000000..439b9ffa8d --- /dev/null +++ b/apps/internal-tool/.env @@ -0,0 +1,9 @@ +# Stack Auth +NEXT_PUBLIC_STACK_API_URL=REPLACE_ME +NEXT_PUBLIC_STACK_PROJECT_ID=REPLACE_ME +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=REPLACE_ME +STACK_SECRET_SERVER_KEY=REPLACE_ME +NEXT_PUBLIC_STACK_DASHBOARD_URL=REPLACE_ME +# SpacetimeDB +NEXT_PUBLIC_SPACETIMEDB_HOST=REPLACE_ME +NEXT_PUBLIC_SPACETIMEDB_DB_NAME=REPLACE_ME diff --git a/apps/internal-tool/.env.development b/apps/internal-tool/.env.development new file mode 100644 index 0000000000..f04eca35e1 --- /dev/null +++ b/apps/internal-tool/.env.development @@ -0,0 +1,6 @@ +NEXT_PUBLIC_STACK_API_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}02 +NEXT_PUBLIC_STACK_PROJECT_ID=internal +NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY=this-publishable-client-key-is-for-local-development-only +NEXT_PUBLIC_STACK_DASHBOARD_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}01 +NEXT_PUBLIC_SPACETIMEDB_HOST=ws://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39 +NEXT_PUBLIC_SPACETIMEDB_DB_NAME=stack-auth-llm diff --git a/apps/internal-tool/.eslintrc.cjs b/apps/internal-tool/.eslintrc.cjs new file mode 100644 index 0000000000..f0b54a7474 --- /dev/null +++ b/apps/internal-tool/.eslintrc.cjs @@ -0,0 +1,7 @@ +module.exports = { + extends: ["../../configs/eslint/defaults.js"], + parserOptions: { + project: "./tsconfig.json", + }, + ignorePatterns: ["/*", "!/src"], +}; diff --git a/apps/internal-tool/next.config.mjs b/apps/internal-tool/next.config.mjs new file mode 100644 index 0000000000..575cb88e2e --- /dev/null +++ b/apps/internal-tool/next.config.mjs @@ -0,0 +1,6 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + poweredByHeader: false, +}; + +export default nextConfig; diff --git a/apps/internal-tool/package.json b/apps/internal-tool/package.json new file mode 100644 index 0000000000..b1ad3dfca7 --- /dev/null +++ b/apps/internal-tool/package.json @@ -0,0 +1,43 @@ +{ + "name": "@stackframe/internal-tool", + "private": true, + "version": "2.8.80", + "type": "module", + "scripts": { + "dev": "node scripts/pre-dev.mjs && next dev --turbopack --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}41", + "build": "next build", + "start": "next start --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}41", + "typecheck": "tsc --noEmit", + "lint": "eslint --ext .ts,.tsx .", + "clean": "rimraf .next && rimraf node_modules", + "spacetime:generate": "spacetime generate --lang typescript --out-dir src/module_bindings --module-path spacetimedb", + "spacetime:inject-token": "node scripts/spacetime-token.mjs inject", + "spacetime:restore-token": "node scripts/spacetime-token.mjs restore", + "spacetime:publish:local": "node scripts/spacetime-publish.mjs local", + "spacetime:publish:prod": "node scripts/spacetime-publish.mjs prod" + }, + "dependencies": { + "@stackframe/stack": "workspace:*", + "@stackframe/stack-shared": "workspace:*", + "clsx": "^2.1.1", + "date-fns": "^4.1.0", + "next": "16.1.5", + "react-markdown": "^10.1.0", + "zod": "^3.24.0", + "remark-gfm": "^4.0.1", + "react": "19.2.3", + "react-dom": "19.2.3", + "spacetimedb": "^2.1.0", + "tailwind-merge": "^3.3.1" + }, + "devDependencies": { + "@types/react": "19.2.7", + "@types/react-dom": "19.2.3", + "autoprefixer": "^10.4.21", + "postcss": "^8.5.4", + "@tailwindcss/typography": "^0.5.16", + "tailwindcss": "^3.4.17", + "typescript": "5.9.3" + }, + "packageManager": "pnpm@10.23.0" +} diff --git a/apps/internal-tool/postcss.config.js b/apps/internal-tool/postcss.config.js new file mode 100644 index 0000000000..2aa7205d4b --- /dev/null +++ b/apps/internal-tool/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/apps/internal-tool/scripts/pre-dev.mjs b/apps/internal-tool/scripts/pre-dev.mjs new file mode 100644 index 0000000000..f8de81e6b0 --- /dev/null +++ b/apps/internal-tool/scripts/pre-dev.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node +// Runs before `next dev`. Publishes the SpacetimeDB module to the local server +// if the spacetime CLI is installed. Otherwise, warns and continues so the +// dev server still starts (useful in CI and for contributors who haven't +// installed the CLI yet). + +import { spawnSync } from "node:child_process"; + +const which = spawnSync(process.platform === "win32" ? "where" : "which", ["spacetime"], { + stdio: "ignore", +}); + +if (which.status !== 0) { + console.warn("\n[internal-tool] spacetime CLI not found, skipping publish."); + console.warn("[internal-tool] To install it: curl -sSf https://install.spacetimedb.com | sh\n"); + process.exit(0); +} + +const publish = spawnSync("pnpm", ["spacetime:publish:local"], { + stdio: "inherit", +}); + +process.exit(publish.status ?? 1); diff --git a/apps/internal-tool/scripts/spacetime-publish.mjs b/apps/internal-tool/scripts/spacetime-publish.mjs new file mode 100644 index 0000000000..6273f91919 --- /dev/null +++ b/apps/internal-tool/scripts/spacetime-publish.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node +// Cross-platform SpacetimeDB publish that injects the token, publishes, and +// always restores the original file — even on failure. + +import { spawnSync } from "node:child_process"; + +const target = process.argv[2]; // "local" or "prod" + +const configs = { + local: ["publish", "stack-auth-llm", "--server", "local", "-p", "spacetimedb", "--yes", "--no-config", "--delete-data=on-conflict"], + prod: ["publish", "stack-auth-llm", "--server", "maincloud", "-p", "spacetimedb", "--yes", "--no-config"], +}; + +const args = configs[target]; +if (!args) { + console.error("Usage: node scripts/spacetime-publish.mjs "); + process.exit(1); +} + +if (target === "prod" && !process.env.STACK_MCP_LOG_TOKEN) { + console.error("Error: STACK_MCP_LOG_TOKEN must be set for prod publish"); + process.exit(1); +} + +let exitCode = 1; +try { + const inject = spawnSync("node", ["scripts/spacetime-token.mjs", "inject"], { stdio: "inherit" }); + if (inject.status !== 0) { + exitCode = inject.status ?? 1; + } else { + const publish = spawnSync("spacetime", args, { stdio: "inherit" }); + exitCode = publish.status ?? 1; + } +} finally { + const restore = spawnSync("node", ["scripts/spacetime-token.mjs", "restore"], { stdio: "inherit" }); + if (restore.status !== 0 && exitCode === 0) { + exitCode = restore.status ?? 1; + } + process.exitCode = exitCode; +} diff --git a/apps/internal-tool/scripts/spacetime-token.mjs b/apps/internal-tool/scripts/spacetime-token.mjs new file mode 100644 index 0000000000..51a8c119ca --- /dev/null +++ b/apps/internal-tool/scripts/spacetime-token.mjs @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Cross-platform token injection/restoration for SpacetimeDB publish. +// Replaces the Unix-only sed/mv scripts so pnpm dev works on Windows too. + +import { readFileSync, writeFileSync, existsSync, renameSync, unlinkSync } from "node:fs"; +import { resolve } from "node:path"; + +const TARGET = resolve("spacetimedb/src/index.ts"); +const BACKUP = TARGET + ".bak"; +const PLACEHOLDER = "__SPACETIMEDB_LOG_TOKEN__"; + +const action = process.argv[2]; + +if (action === "inject") { + const token = process.env.STACK_MCP_LOG_TOKEN || "change-me"; + if (existsSync(BACKUP)) { + console.error("Refusing to inject: backup already exists. Run restore first."); + process.exit(1); + } + const content = readFileSync(TARGET, "utf8"); + writeFileSync(BACKUP, content, "utf8"); + const escapedToken = JSON.stringify(token).slice(1, -1); + writeFileSync(TARGET, content.replaceAll(PLACEHOLDER, escapedToken), "utf8"); +} else if (action === "restore") { + if (existsSync(BACKUP)) { + if (existsSync(TARGET)) { + unlinkSync(TARGET); + } + renameSync(BACKUP, TARGET); + } +} else { + console.error("Usage: node scripts/spacetime-token.mjs "); + process.exit(1); +} diff --git a/apps/internal-tool/spacetime.json b/apps/internal-tool/spacetime.json new file mode 100644 index 0000000000..11f5588437 --- /dev/null +++ b/apps/internal-tool/spacetime.json @@ -0,0 +1,6 @@ +{ + "dev": { + "run": "npm run dev" + }, + "module_path": "spacetimedb" +} \ No newline at end of file diff --git a/apps/internal-tool/spacetimedb/package.json b/apps/internal-tool/spacetimedb/package.json new file mode 100644 index 0000000000..5b11f15b53 --- /dev/null +++ b/apps/internal-tool/spacetimedb/package.json @@ -0,0 +1,12 @@ +{ + "name": "@stackframe/internal-tool-module", + "private": true, + "version": "2.8.80", + "type": "module", + "dependencies": { + "spacetimedb": "^2.1.0" + }, + "devDependencies": { + "typescript": "5.9.3" + } +} diff --git a/apps/internal-tool/spacetimedb/src/index.ts b/apps/internal-tool/spacetimedb/src/index.ts new file mode 100644 index 0000000000..da052f4bf7 --- /dev/null +++ b/apps/internal-tool/spacetimedb/src/index.ts @@ -0,0 +1,240 @@ +import { schema, t, table, SenderError } from 'spacetimedb/server'; + +// Injected at publish time by the spacetime:inject-token pnpm script from STACK_MCP_LOG_TOKEN env var. +// Must match STACK_MCP_LOG_TOKEN in the backend .env. +const EXPECTED_LOG_TOKEN = '__SPACETIMEDB_LOG_TOKEN__'; + +const mcpCallLog = table( + { name: 'mcp_call_log', public: true }, + { + id: t.u64().primaryKey().autoInc(), + correlationId: t.string(), + conversationId: t.string().optional(), + createdAt: t.timestamp(), + toolName: t.string(), + reason: t.string(), + userPrompt: t.string(), + question: t.string(), + response: t.string(), + stepCount: t.u32(), + innerToolCallsJson: t.string(), + durationMs: t.u64(), + modelId: t.string(), + errorMessage: t.string().optional(), + // QA review fields (populated asynchronously after initial log) + qaReviewedAt: t.timestamp().optional(), + qaNeedsHumanReview: t.bool().optional(), + qaAnswerCorrect: t.bool().optional(), + qaAnswerRelevant: t.bool().optional(), + qaFlagsJson: t.string().optional(), + qaImprovementSuggestions: t.string().optional(), + qaOverallScore: t.u32().optional(), + qaReviewModelId: t.string().optional(), + qaConversationJson: t.string().optional(), + qaErrorMessage: t.string().optional(), + // Human review + humanReviewedAt: t.timestamp().optional(), + humanReviewedBy: t.string().optional(), + // Human corrections & publishing + humanCorrectedQuestion: t.string().optional(), + humanCorrectedAnswer: t.string().optional(), + publishedToQa: t.bool().optional(), + publishedAt: t.timestamp().optional(), + } +); + +const spacetimedb = schema({ mcpCallLog }); +export default spacetimedb; + +export const log_mcp_call = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + conversationId: t.string().optional(), + toolName: t.string(), + reason: t.string(), + userPrompt: t.string(), + question: t.string(), + response: t.string(), + stepCount: t.u32(), + innerToolCallsJson: t.string(), + durationMs: t.u64(), + modelId: t.string(), + errorMessage: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.mcpCallLog.insert({ + id: 0n, + correlationId: args.correlationId, + conversationId: args.conversationId, + createdAt: ctx.timestamp, + toolName: args.toolName, + reason: args.reason, + userPrompt: args.userPrompt, + question: args.question, + response: args.response, + stepCount: args.stepCount, + innerToolCallsJson: args.innerToolCallsJson, + durationMs: args.durationMs, + modelId: args.modelId, + errorMessage: args.errorMessage, + } as Parameters[0]); + } +); + +export const update_mcp_qa_review = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + qaNeedsHumanReview: t.bool(), + qaAnswerCorrect: t.bool(), + qaAnswerRelevant: t.bool(), + qaFlagsJson: t.string(), + qaImprovementSuggestions: t.string(), + qaOverallScore: t.u32(), + qaReviewModelId: t.string(), + qaConversationJson: t.string().optional(), + qaErrorMessage: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + qaReviewedAt: ctx.timestamp, + qaNeedsHumanReview: args.qaNeedsHumanReview, + qaAnswerCorrect: args.qaAnswerCorrect, + qaAnswerRelevant: args.qaAnswerRelevant, + qaFlagsJson: args.qaFlagsJson, + qaImprovementSuggestions: args.qaImprovementSuggestions, + qaOverallScore: args.qaOverallScore, + qaReviewModelId: args.qaReviewModelId, + qaConversationJson: args.qaConversationJson, + qaErrorMessage: args.qaErrorMessage, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const mark_human_reviewed = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + reviewedBy: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + humanReviewedAt: ctx.timestamp, + humanReviewedBy: args.reviewedBy, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const update_human_correction = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + correctedQuestion: t.string(), + correctedAnswer: t.string(), + publish: t.bool(), + reviewedBy: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + ctx.db.mcpCallLog.insert({ + ...row, + humanCorrectedQuestion: args.correctedQuestion, + humanCorrectedAnswer: args.correctedAnswer, + humanReviewedAt: row.humanReviewedAt ?? ctx.timestamp, + humanReviewedBy: row.humanReviewedBy ?? args.reviewedBy, + publishedToQa: args.publish, + publishedAt: args.publish ? (row.publishedAt ?? ctx.timestamp) : undefined, + }); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const add_manual_qa = spacetimedb.reducer( + { + token: t.string(), + question: t.string(), + answer: t.string(), + publish: t.bool(), + reviewedBy: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.mcpCallLog.insert({ + id: 0n, + correlationId: ctx.newUuidV4().toString(), + createdAt: ctx.timestamp, + toolName: "manual", + reason: "Manually added Q&A", + userPrompt: "", + question: args.question, + response: "", + stepCount: 0, + innerToolCallsJson: "[]", + durationMs: 0n, + modelId: "human", + humanCorrectedQuestion: args.question, + humanCorrectedAnswer: args.answer, + humanReviewedAt: ctx.timestamp, + humanReviewedBy: args.reviewedBy, + publishedToQa: args.publish, + publishedAt: args.publish ? ctx.timestamp : undefined, + } as Parameters[0]); + } +); + +export const delete_qa_entry = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const row of ctx.db.mcpCallLog.iter()) { + if (row.correlationId === args.correlationId) { + ctx.db.mcpCallLog.delete(row); + return; + } + } + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } +); + +export const init = spacetimedb.init(_ctx => {}); diff --git a/apps/internal-tool/spacetimedb/tsconfig.json b/apps/internal-tool/spacetimedb/tsconfig.json new file mode 100644 index 0000000000..93673d13c0 --- /dev/null +++ b/apps/internal-tool/spacetimedb/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist" + }, + "include": ["src"] +} diff --git a/apps/internal-tool/src/app/app-client.tsx b/apps/internal-tool/src/app/app-client.tsx new file mode 100644 index 0000000000..d360c260ad --- /dev/null +++ b/apps/internal-tool/src/app/app-client.tsx @@ -0,0 +1,198 @@ +import { useUser } from "@stackframe/stack"; +import { clsx } from "clsx"; +import { useState } from "react"; +import { AddManualQa } from "../components/AddManualQa"; +import { Analytics } from "../components/Analytics"; +import { CallLogDetail } from "../components/CallLogDetail"; +import { CallLogList } from "../components/CallLogList"; +import { KnowledgeBase } from "../components/KnowledgeBase"; +import { useMcpCallLogs } from "../hooks/useSpacetimeDB"; +import { makeMcpReviewApi } from "../lib/mcp-review-api"; +import type { McpCallLogRow } from "../types"; + +type Tab = "calls" | "knowledge" | "analytics"; + +export default function App() { + const user = useUser({ or: process.env.NODE_ENV === "development" ? "redirect" : "return-null" }); + const [selectedRow, setSelectedRow] = useState(null); + const [showAddQa, setShowAddQa] = useState(false); + const [tab, setTab] = useState("calls"); + const { rows, connectionState } = useMcpCallLogs(); + + if (!user) { + return ( +
+
+

MCP Review Tool

+

+ Sign in to the{" "} + + Stack Dashboard + + {" "}first, then reload this page. +

+ +
+
+ ); + } + + if (process.env.NODE_ENV !== "development") { + const metadata = user.clientReadOnlyMetadata as Record | null; + if (!metadata?.isAiChatReviewer) { + return ( +
+
+

Access Denied

+

+ You are signed in as {user.displayName ?? user.primaryEmail}, but your account is not approved. +

+
+
+ ); + } + } + + const currentSelectedRow = selectedRow + ? rows.find(r => r.id === selectedRow.id) ?? selectedRow + : null; + + const currentUser = user; + + async function getApi() { + const { accessToken, refreshToken } = await currentUser.getAuthJson(); + const authHeaders: Record = {}; + if (accessToken) authHeaders["x-stack-access-token"] = accessToken; + if (refreshToken) authHeaders["x-stack-refresh-token"] = refreshToken; + return makeMcpReviewApi(authHeaders); + } + + return ( +
+
+
+

MCP Review Tool

+ {/* Tabs */} +
+ + + +
+
+
+ + {user.displayName ?? user.primaryEmail} +
+
+ + {showAddQa && ( + setShowAddQa(false)} + onSave={async (question, answer, publish) => { + const api = await getApi(); + await api.addManual({ question, answer, publish }); + }} + /> + )} + + {tab === "calls" && ( +
+
+ +
+ {currentSelectedRow && ( + + )} +
+ )} + + {tab === "knowledge" && ( +
+ { + getApi() + .then(api => api.updateCorrection({ correlationId, correctedQuestion: question, correctedAnswer: answer, publish })) + .catch(() => { /* errors are surfaced by UI state */ }); + }} + onDelete={(correlationId) => { + getApi() + .then(api => api.delete({ correlationId })) + .catch(() => { /* errors are surfaced by UI state */ }); + }} + /> +
+ )} + + {tab === "analytics" && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/internal-tool/src/app/globals.css b/apps/internal-tool/src/app/globals.css new file mode 100644 index 0000000000..7012e083ef --- /dev/null +++ b/apps/internal-tool/src/app/globals.css @@ -0,0 +1,10 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} diff --git a/apps/internal-tool/src/app/handler/[...stack]/page.tsx b/apps/internal-tool/src/app/handler/[...stack]/page.tsx new file mode 100644 index 0000000000..59ae576f3e --- /dev/null +++ b/apps/internal-tool/src/app/handler/[...stack]/page.tsx @@ -0,0 +1,11 @@ +"use client"; + +import { StackHandler } from "@stackframe/stack"; +import { useEffect, useState } from "react"; + +export default function Handler() { + const [mounted, setMounted] = useState(false); + useEffect(() => setMounted(true), []); + if (!mounted) return null; + return ; +} diff --git a/apps/internal-tool/src/app/layout.tsx b/apps/internal-tool/src/app/layout.tsx new file mode 100644 index 0000000000..d0dd7d4dc3 --- /dev/null +++ b/apps/internal-tool/src/app/layout.tsx @@ -0,0 +1,26 @@ +"use client"; + +import { Suspense } from "react"; +import { StackProvider, StackTheme } from "@stackframe/stack"; +import { stackClientApp } from "../stack"; +import Loading from "./loading"; +import "./globals.css"; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + + Stack Auth — MCP Review Tool + + + + + }> + {children} + + + + + + ); +} diff --git a/apps/internal-tool/src/app/loading.tsx b/apps/internal-tool/src/app/loading.tsx new file mode 100644 index 0000000000..ebfed0abd9 --- /dev/null +++ b/apps/internal-tool/src/app/loading.tsx @@ -0,0 +1,10 @@ +export default function Loading() { + return ( +
+
+
+

Loading...

+
+
+ ); +} diff --git a/apps/internal-tool/src/app/page.tsx b/apps/internal-tool/src/app/page.tsx new file mode 100644 index 0000000000..7e07994b82 --- /dev/null +++ b/apps/internal-tool/src/app/page.tsx @@ -0,0 +1,7 @@ +"use client"; + +import App from "./app-client"; + +export default function Page() { + return ; +} diff --git a/apps/internal-tool/src/app/questions/page.tsx b/apps/internal-tool/src/app/questions/page.tsx new file mode 100644 index 0000000000..2fcadcc2d6 --- /dev/null +++ b/apps/internal-tool/src/app/questions/page.tsx @@ -0,0 +1,71 @@ +"use client"; + +import { useMemo } from "react"; +import Markdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import { format } from "date-fns"; +import { useMcpCallLogs } from "../../hooks/useSpacetimeDB"; +import { toDate } from "../../utils"; +import { markdownComponents } from "../../components/markdown-components"; + +export default function QuestionsPage() { + const { rows, connectionState } = useMcpCallLogs(); + + const publishedQa = useMemo(() => { + return rows + .filter(r => r.publishedToQa) + .sort((a, b) => { + const aTime = a.publishedAt ? Number(toDate(a.publishedAt)) : 0; + const bTime = b.publishedAt ? Number(toDate(b.publishedAt)) : 0; + return bTime - aTime; + }); + }, [rows]); + + if (connectionState === "connecting") { + return ( +
+

Loading...

+
+ ); + } + + if (connectionState === "error") { + return ( +
+

Failed to connect to database.

+
+ ); + } + + return ( +
+

Stack Auth Q&A

+

+ Curated questions and answers about Stack Auth, reviewed by humans. +

+ + {publishedQa.length === 0 ? ( +

No published Q&A yet.

+ ) : ( +
+ {publishedQa.map(row => ( +
+

{row.humanCorrectedQuestion ?? row.question}

+
+ + {row.humanCorrectedAnswer ?? row.response} + +
+
+ {row.toolName} + {row.publishedAt && ( + {format(toDate(row.publishedAt), "MMM d, yyyy")} + )} +
+
+ ))} +
+ )} +
+ ); +} diff --git a/apps/internal-tool/src/components/AddManualQa.tsx b/apps/internal-tool/src/components/AddManualQa.tsx new file mode 100644 index 0000000000..faa19b0521 --- /dev/null +++ b/apps/internal-tool/src/components/AddManualQa.tsx @@ -0,0 +1,116 @@ +import { useState } from "react"; +import { clsx } from "clsx"; + +export function AddManualQa({ onClose, onSave }: { + onClose: () => void; + onSave: (question: string, answer: string, publish: boolean) => Promise; +}) { + const [question, setQuestion] = useState(""); + const [answer, setAnswer] = useState(""); + const [saved, setSaved] = useState(false); + const [isSaving, setIsSaving] = useState(false); + const [error, setError] = useState(null); + + const canSave = question.trim().length > 0 && answer.trim().length > 0 && !isSaving; + + const handleSave = async (publish: boolean) => { + if (!canSave) return; + setIsSaving(true); + setError(null); + try { + await onSave(question.trim(), answer.trim(), publish); + setSaved(true); + setTimeout(() => { + setSaved(false); + setQuestion(""); + setAnswer(""); + if (publish) { + onClose(); + } + }, 1500); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to save"); + } finally { + setIsSaving(false); + } + }; + + return ( +
+
+ {/* Header */} +
+

Add Q&A

+ +
+ + {/* Form */} +
+ {saved && ( +
+ Saved successfully +
+ )} + {error && ( +
+ {error} +
+ )} + +
+ + setQuestion(e.target.value)} + placeholder="e.g. How do I set up OAuth with Stack Auth?" + /> +
+ +
+ +