LLM MCP Flow

.github/workflows/db-migration-backwards-compatibility.yaml

@@ -138,6 +138,9 @@ jobs:
       - name: Create .env.test.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.test.local
 
+      - name: Create .env.test.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
+
       - name: Build
         run: pnpm build
 
@@ -332,6 +335,9 @@ jobs:
       - name: Create .env.test.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.test.local
 
+      - name: Create .env.test.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
+
       - name: Build
         run: pnpm build
 

.github/workflows/e2e-api-tests-local-emulator.yaml

@@ -86,6 +86,9 @@ jobs:
       - name: Create .env.test.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.test.local
 
+      - name: Create .env.test.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
+
       - name: Build
         run: pnpm build
 

.github/workflows/e2e-api-tests.yaml

@@ -92,6 +92,9 @@ jobs:
       - name: Create .env.test.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.test.local
 
+      - name: Create .env.test.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
+
       - name: Build
         run: pnpm build
 

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -85,6 +85,9 @@ jobs:
       - name: Create .env.test.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.test.local
 
+      - name: Create .env.test.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
+
       - name: Build
         run: pnpm build
 

.github/workflows/e2e-fallback-tests.yaml

@@ -68,6 +68,7 @@ jobs:
           cp examples/middleware/.env.development examples/middleware/.env.test.local
           cp examples/supabase/.env.development examples/supabase/.env.test.local
           cp examples/convex/.env.development examples/convex/.env.test.local
+          cp apps/internal-tool/.env.development apps/internal-tool/.env.test.local
 
       - name: Build
         run: pnpm build

.github/workflows/lint-and-build.yaml

@@ -66,6 +66,9 @@ jobs:
       - name: Create .env.production.local file for examples/convex
         run: cp examples/convex/.env.development examples/convex/.env.production.local
 
+      - name: Create .env.production.local file for apps/internal-tool
+        run: cp apps/internal-tool/.env.development apps/internal-tool/.env.production.local
+
       - name: Build
         run: pnpm build
 

README.md

@@ -1,5 +1,7 @@
 [![Stack Logo](/.github/assets/logo.png)](https://stack-auth.com)
 
+[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/stack-auth/stack-auth)
+
 <h3 align="center">
   <a href="https://docs.stack-auth.com">📘 Docs</a>
   | <a href="https://stack-auth.com/">☁️ Hosted Version</a>

apps/backend/.env

@@ -118,3 +118,8 @@ STACK_TELEGRAM_CHAT_ID=# enter your telegram chat id
 
 # Docs AI tool bundle
 STACK_DOCS_INTERNAL_BASE_URL=# override the docs origin used by the backend's AI tool bundle to call the docs app's `/api/internal/docs-tools` endpoint. Defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod
+
+# MCP review tool (SpacetimeDB)
+STACK_SPACETIMEDB_URI=# SpacetimeDB host URI; default empty (logging disabled)
+STACK_SPACETIMEDB_DB_NAME=# SpacetimeDB database name
+STACK_MCP_LOG_TOKEN=# shared secret gating the log_mcp_call reducer; must match EXPECTED_LOG_TOKEN in apps/internal-tool/spacetimedb/src/index.ts

apps/backend/.env.development

@@ -113,6 +113,11 @@ STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc
 STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
 STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
 
+# MCP review tool (SpacetimeDB)
+STACK_SPACETIMEDB_URI=ws://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39
+STACK_SPACETIMEDB_DB_NAME=stack-auth-llm
+STACK_MCP_LOG_TOKEN=change-me
+
 # Clickhouse
 STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36
 STACK_CLICKHOUSE_ADMIN_USER=stackframe

apps/backend/package.json

@@ -55,6 +55,7 @@
   },
   "dependencies": {
     "@ai-sdk/mcp": "^1.0.21",
+    "spacetimedb": "^2.1.0",
     "@ai-sdk/openai": "^3.0.29",
     "@aws-sdk/client-s3": "^3.855.0",
     "@clickhouse/client": "^1.14.0",

apps/backend/src/app/api/latest/ai/query/[mode]/route.ts

@@ -1,10 +1,14 @@
+import { logMcpCall } from "@/lib/ai/mcp-logger";
 import { selectModel } from "@/lib/ai/models";
 import { getFullSystemPrompt } from "@/lib/ai/prompts";
+import { reviewMcpCall } from "@/lib/ai/qa-reviewer";
 import { requestBodySchema } from "@/lib/ai/schema";
 import { getTools, validateToolNames } from "@/lib/ai/tools";
+import { getVerifiedQaContext } from "@/lib/ai/verified-qa";
 import { listManagedProjectIds } from "@/lib/projects";
 import { SmartResponse } from "@/route-handlers/smart-response";
 import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks";
 import { validateImageAttachments } from "@stackframe/stack-shared/dist/ai/image-limits";
 import { yupMixed, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
 import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
@@ -52,15 +56,14 @@ export const POST = createSmartRouteHandler({
     }
 
     const model = selectModel(quality, speed, isAuthenticated);
-    const systemPrompt = getFullSystemPrompt(systemPromptId);
+    const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai";
+    let systemPrompt = getFullSystemPrompt(systemPromptId);
+    if (isDocsOrSearch) {
+      systemPrompt += await getVerifiedQaContext();
+    }
     const tools = await getTools(toolNames, { auth: fullReq.auth, targetProjectId: projectId });
     const toolsArg = Object.keys(tools).length > 0 ? tools : undefined;
-    const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai";
-    // create-dashboard now does an inspection loop (queryAnalytics) before calling updateDashboard,
-    // so it needs room for ~3 exploratory queries + the final tool call + some retry slack.
     const isCreateDashboard = systemPromptId === "create-dashboard";
-    // build-analytics-query aims for one-shot queries with complete schema
-    // knowledge, but needs a few steps for retries on errors or follow-ups.
     const isBuildAnalyticsQuery = systemPromptId === "build-analytics-query";
     const stepLimit = toolsArg == null
       ? 1
@@ -86,6 +89,7 @@ export const POST = createSmartRouteHandler({
         body: result.toUIMessageStreamResponse(),
       };
     } else {
+      const startedAt = Date.now();
       const controller = new AbortController();
       const timeoutId = setTimeout(() => controller.abort(), 120_000);
       const result = await generateText({
@@ -134,10 +138,51 @@ export const POST = createSmartRouteHandler({
         });
       });
 
+      let responseConversationId: string | undefined;
+      if (body.mcpCallMetadata != null) {
+        const correlationId = crypto.randomUUID();
+        const conversationId = body.mcpCallMetadata.conversationId ?? crypto.randomUUID();
+        responseConversationId = conversationId;
+        const firstUserMessage = messages.find(m => m.role === "user");
+        const question = typeof firstUserMessage?.content === "string"
+          ? firstUserMessage.content
+          : JSON.stringify(firstUserMessage?.content ?? "");
+
+        const innerToolCallsJson = JSON.stringify(contentBlocks.filter(b => b.type === "tool-call"));
+
+        const logPromise = logMcpCall({
+          correlationId,
+          toolName: body.mcpCallMetadata.toolName,
+          reason: body.mcpCallMetadata.reason,
+          userPrompt: body.mcpCallMetadata.userPrompt,
+          conversationId,
+          question,
+          response: result.text,
+          stepCount: result.steps.length,
+          innerToolCallsJson,
+          durationMs: BigInt(Date.now() - startedAt),
+          modelId: String(model.modelId),
+          errorMessage: undefined,
+        });
+        runAsynchronouslyAndWaitUntil(logPromise);
+
+        runAsynchronouslyAndWaitUntil(reviewMcpCall({
+          logPromise,
+          correlationId,
+          question,
+          reason: body.mcpCallMetadata.reason,
+          response: result.text,
+        }));
+      }
+
       return {
         statusCode: 200,
         bodyType: "json" as const,
-        body: { content: contentBlocks, finalText: result.text },
+        body: {
+          content: contentBlocks,
+          finalText: result.text,
+          conversationId: responseConversationId ?? null,
+        },
       };
     }
   },

apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts

@@ -0,0 +1,55 @@
+import { getConnectionOrThrow } from "@/lib/ai/mcp-logger";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
+import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+
+export const POST = createSmartRouteHandler({
+  metadata: { hidden: true },
+  request: yupObject({
+    auth: yupObject({
+      type: adaptSchema,
+      user: adaptSchema.defined(),
+      project: adaptSchema,
+    }).defined(),
+    body: yupObject({
+      question: yupString().defined(),
+      answer: yupString().defined(),
+      publish: yupBoolean().defined(),
+    }).defined(),
+    method: yupString().oneOf(["POST"]).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: yupObject({
+      success: yupBoolean().defined(),
+    }).defined(),
+  }),
+  handler: async ({ auth, body }) => {
+    const user = auth.user;
+    if (getNodeEnvironment() !== "development") {
+      const metadata = user.client_read_only_metadata;
+      if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) {
+        throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations.");
+      }
+    }
+
+    const conn = await getConnectionOrThrow();
+
+    const token = getEnvVariable("STACK_MCP_LOG_TOKEN");
+    await conn.reducers.addManualQa({
+      token,
+      question: body.question,
+      answer: body.answer,
+      publish: body.publish,
+      reviewedBy: user.display_name ?? user.primary_email ?? user.id,
+    });
+
+    return {
+      statusCode: 200,
+      bodyType: "json" as const,
+      body: { success: true },
+    };
+  },
+});

apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts

@@ -0,0 +1,49 @@
+import { getConnectionOrThrow } from "@/lib/ai/mcp-logger";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
+import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+
+export const POST = createSmartRouteHandler({
+  metadata: { hidden: true },
+  request: yupObject({
+    auth: yupObject({
+      type: adaptSchema,
+      user: adaptSchema.defined(),
+      project: adaptSchema,
+    }).defined(),
+    body: yupObject({
+      correlationId: yupString().defined(),
+    }).defined(),
+    method: yupString().oneOf(["POST"]).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: yupObject({
+      success: yupBoolean().defined(),
+    }).defined(),
+  }),
+  handler: async ({ auth, body }) => {
+    if (getNodeEnvironment() !== "development") {
+      const metadata = auth.user.client_read_only_metadata;
+      if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) {
+        throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations.");
+      }
+    }
+
+    const conn = await getConnectionOrThrow();
+
+    const token = getEnvVariable("STACK_MCP_LOG_TOKEN");
+    await conn.reducers.deleteQaEntry({
+      token,
+      correlationId: body.correlationId,
+    });
+
+    return {
+      statusCode: 200,
+      bodyType: "json" as const,
+      body: { success: true },
+    };
+  },
+});

apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts

@@ -0,0 +1,51 @@
+import { getConnectionOrThrow } from "@/lib/ai/mcp-logger";
+import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
+import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
+import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+
+export const POST = createSmartRouteHandler({
+  metadata: { hidden: true },
+  request: yupObject({
+    auth: yupObject({
+      type: adaptSchema,
+      user: adaptSchema.defined(),
+      project: adaptSchema,
+    }).defined(),
+    body: yupObject({
+      correlationId: yupString().defined(),
+    }).defined(),
+    method: yupString().oneOf(["POST"]).defined(),
+  }),
+  response: yupObject({
+    statusCode: yupNumber().oneOf([200]).defined(),
+    bodyType: yupString().oneOf(["json"]).defined(),
+    body: yupObject({
+      success: yupBoolean().defined(),
+    }).defined(),
+  }),
+  handler: async ({ auth, body }) => {
+    const user = auth.user;
+    if (getNodeEnvironment() !== "development") {
+      const metadata = user.client_read_only_metadata;
+      if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) {
+        throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations.");
+      }
+    }
+
+    const conn = await getConnectionOrThrow();
+
+    const token = getEnvVariable("STACK_MCP_LOG_TOKEN");
+    await conn.reducers.markHumanReviewed({
+      token,
+      correlationId: body.correlationId,
+      reviewedBy: user.display_name ?? user.primary_email ?? user.id,
+    });
+
+    return {
+      statusCode: 200,
+      bodyType: "json" as const,
+      body: { success: true },
+    };
+  },
+});