diff --git a/apps/backend/.env b/apps/backend/.env index 228c065825..2fb7842ecb 100644 --- a/apps/backend/.env +++ b/apps/backend/.env @@ -120,6 +120,7 @@ STACK_TELEGRAM_CHAT_ID=# enter your telegram chat id STACK_MINTLIFY_MCP_URL=# override the Mintlify MCP server used by the backend's AI docs tool bundle. Defaults to https://stackauth-e0affa27.mintlify.app/mcp # MCP review tool (SpacetimeDB) -STACK_SPACETIMEDB_URI=# SpacetimeDB host URI; default empty (logging disabled) +STACK_SPACETIMEDB_URL=# SpacetimeDB host URL; default empty (logging disabled) STACK_SPACETIMEDB_DB_NAME=# SpacetimeDB database name STACK_MCP_LOG_TOKEN=# shared secret gating the log_mcp_call reducer; must match EXPECTED_LOG_TOKEN in apps/internal-tool/spacetimedb/src/index.ts +STACK_SPACETIMEDB_SERVICE_TOKEN=# backend's SpacetimeDB-minted identity token diff --git a/apps/backend/.env.development b/apps/backend/.env.development index f20581e078..8c566273c4 100644 --- a/apps/backend/.env.development +++ b/apps/backend/.env.development @@ -114,10 +114,14 @@ STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs # MCP review tool (SpacetimeDB) -STACK_SPACETIMEDB_URI=ws://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39 +STACK_SPACETIMEDB_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39 STACK_SPACETIMEDB_DB_NAME=stack-auth-llm STACK_MCP_LOG_TOKEN=change-me +# To provision locally: `curl -X POST http://127.0.0.1:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}39/v1/identity` +# then copy the `token` field from the response. +STACK_SPACETIMEDB_SERVICE_TOKEN= + # Clickhouse STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36 STACK_CLICKHOUSE_ADMIN_USER=stackframe diff --git a/apps/backend/prisma/seed.ts b/apps/backend/prisma/seed.ts index 6081d98af0..9b6a3c520f 100644 --- a/apps/backend/prisma/seed.ts +++ b/apps/backend/prisma/seed.ts @@ -425,6 +425,19 @@ export async function seed() { } } + const existingDefaultUser = await usersCrudHandlers.adminRead({ + tenancy: internalTenancy, + user_id: defaultUserId, + }); + const existingMetadata = (existingDefaultUser.client_read_only_metadata ?? {}) as Record; + await usersCrudHandlers.adminUpdate({ + tenancy: internalTenancy, + user_id: defaultUserId, + data: { + client_read_only_metadata: { ...existingMetadata, isAiChatReviewer: true }, + }, + }); + // Create or ensure TeamMember exists before granting permissions. // Using upsert here (instead of create inside the else block above) ensures // idempotency when adminInternalAccess changes between seed runs. diff --git a/apps/backend/src/app/api/internal/[transport]/route.ts b/apps/backend/src/app/api/internal/[transport]/route.ts index 7b5880ba06..8b0dbacf43 100644 --- a/apps/backend/src/app/api/internal/[transport]/route.ts +++ b/apps/backend/src/app/api/internal/[transport]/route.ts @@ -81,10 +81,14 @@ const handler = createMcpHandler( .join("\n\n") ?? ""; - const responseConversationId = body.conversationId ?? conversationId ?? ""; + const responseConversationId = body.conversationId ?? conversationId; + const bodyText = text.length > 0 ? text : "(empty response)"; + const fullText = responseConversationId + ? `${bodyText}\n\n[conversationId: ${responseConversationId} — pass this value as the conversationId parameter in your next ask_stack_auth call to continue this conversation]` + : bodyText; return { - content: [{ type: "text", text: `${text.length > 0 ? text : "(empty response)"}\n\n[conversationId: ${responseConversationId} - pass this value as the conversationId parameter in your next ask_stack_auth call to continue this conversation]` }], + content: [{ type: "text", text: fullText }], }; }, ); diff --git a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts index 62178ccdbc..b8ee7e0b14 100644 --- a/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts +++ b/apps/backend/src/app/api/latest/ai/query/[mode]/route.ts @@ -1,21 +1,44 @@ -import { logMcpCall } from "@/lib/ai/mcp-logger"; +import { + assertProjectAccess, + handleGenerateMode, + handleStreamMode, +} from "@/lib/ai/ai-query-handlers"; +import type { CommonLogFields, ModeContext } from "@/lib/ai/types"; import { selectModel } from "@/lib/ai/models"; -import { getFullSystemPrompt } from "@/lib/ai/prompts"; -import { reviewMcpCall } from "@/lib/ai/qa-reviewer"; +import { getFullSystemPrompt, type SystemPromptId } from "@/lib/ai/prompts"; import { requestBodySchema } from "@/lib/ai/schema"; import { getTools } from "@/lib/ai/tools"; -import { getVerifiedQaContext } from "@/lib/ai/verified-qa"; -import { listManagedProjectIds } from "@/lib/projects"; +import { getVerifiedQaContext } from "@/lib/ai/qa/verified-qa"; import { SmartResponse } from "@/route-handlers/smart-response"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; -import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; import { validateImageAttachments } from "@stackframe/stack-shared/dist/ai/image-limits"; -import { ChatContent } from "@stackframe/stack-shared/dist/interface/admin-interface"; +import { KnownErrors } from "@stackframe/stack-shared/dist/known-errors"; import { yupMixed, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; -import { Json } from "@stackframe/stack-shared/dist/utils/json"; -import { generateText, stepCountIs, streamText, type ModelMessage } from "ai"; +import type { ModelMessage } from "ai"; + +function getStepLimit(systemPromptId: SystemPromptId, hasTools: boolean): number { + if (!hasTools) return 1; + if (systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai") return 50; + if (systemPromptId === "create-dashboard") return 12; + return 5; +} + +async function buildSystemPrompt(systemPromptId: SystemPromptId): Promise { + let systemPrompt = getFullSystemPrompt(systemPromptId); + const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; + if (isDocsOrSearch) { + // Stuffing the entire verified QA corpus into the system prompt on every + // request is intentionally naive — it grows monotonically with each new + // QA pair and re-fetches/re-sends content that's unchanged across + // requests. Once the corpus is large enough to matter we should swap to + // a retriever based system (maybe something like an embedding-based retriever + // (top-k by query similarity)) and/or cache the assembled context, + // but for the current corpus size this is fine and lets the model see everything + systemPrompt += await getVerifiedQaContext(); + } + return systemPrompt; +} export const POST = createSmartRouteHandler({ metadata: { @@ -34,144 +57,80 @@ export const POST = createSmartRouteHandler({ const { quality, speed, systemPrompt: systemPromptId, tools: toolNames, messages, projectId } = body; if (projectId != null) { - if (fullReq.auth?.project.id !== "internal") { - throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); - } - const user = fullReq.auth.user; - if (user == null) { - throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); - } - const managedProjectIds = await listManagedProjectIds(user); - if (!managedProjectIds.includes(projectId)) { - throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); - } + await assertProjectAccess(projectId, fullReq.auth); } - const imageValidationResult = validateImageAttachments(messages); if (!imageValidationResult.ok) { - throw new StatusError(StatusError.BadRequest, imageValidationResult.reason); + const { failure } = imageValidationResult; + switch (failure.code) { + case "too_many": { + throw new KnownErrors.TooManyImageAttachments(failure.maxImages); + } + case "too_large": { + throw new KnownErrors.ImageAttachmentTooLarge(failure.maxBytes, failure.actualBytes); + } + } } const authenticatedApiKey = isAuthenticated ? getEnvVariable("STACK_OPENROUTER_AUTHENTICATED_API_KEY", "") : ""; const model = selectModel(quality, speed, isAuthenticated, authenticatedApiKey || undefined); - const isDocsOrSearch = systemPromptId === "docs-ask-ai" || systemPromptId === "command-center-ask-ai"; - let systemPrompt = getFullSystemPrompt(systemPromptId); - if (isDocsOrSearch) { - systemPrompt += await getVerifiedQaContext(); - } + const systemPrompt = await buildSystemPrompt(systemPromptId); const tools = await getTools(toolNames, { auth: fullReq.auth, targetProjectId: projectId }); const toolsArg = Object.keys(tools).length > 0 ? tools : undefined; - const isCreateDashboard = systemPromptId === "create-dashboard"; - const isBuildAnalyticsQuery = systemPromptId === "build-analytics-query"; - const stepLimit = toolsArg == null - ? 1 - : isDocsOrSearch - ? 50 - : isCreateDashboard - ? 12 - : isBuildAnalyticsQuery - ? 5 - : 5; + const stepLimit = getStepLimit(systemPromptId, toolsArg != null); + + const correlationId = crypto.randomUUID(); + const conversationIdForLog = body.mcpCallMetadata + ? body.mcpCallMetadata.conversationId ?? crypto.randomUUID() + : undefined; + const common: CommonLogFields = { + correlationId, + mode, + systemPromptId, + quality, + speed, + modelId: String(model.modelId), + isAuthenticated, + projectId: projectId ?? undefined, + userId: fullReq.auth?.user?.id, + requestedToolsJson: JSON.stringify(toolNames), + messagesJson: JSON.stringify(messages), + conversationId: conversationIdForLog, + }; + const startedAt = performance.now(); + const isAnthropic = model.modelId.startsWith("anthropic/"); + // Can be optimized: only opt into prompt caching for routes that are hit + // frequently enough to amortize the write. + const systemMessage: ModelMessage = { + role: "system", + content: systemPrompt, + ...(isAnthropic && { + providerOptions: { + openrouter: { cacheControl: { type: "ephemeral" } }, + }, + }), + }; // Cast: the schema narrows role and leaves content as unknown, but the // AI SDK accepts a superset (role: "system" etc.). We've intentionally // excluded `system` at the schema layer to prevent prompt-injection via // client-supplied system messages — see schema.ts. const modelMessages = messages as unknown as ModelMessage[]; + const cachedMessages: ModelMessage[] = [systemMessage, ...modelMessages]; - if (mode === "stream") { - const result = streamText({ - model, - system: systemPrompt, - messages: modelMessages, - tools: toolsArg, - stopWhen: stepCountIs(stepLimit), - }); - return { - statusCode: 200, - bodyType: "response" as const, - body: result.toUIMessageStreamResponse(), - }; - } else { - const startedAt = Date.now(); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 120_000); - const result = await generateText({ - model, - system: systemPrompt, - messages: modelMessages, - tools: toolsArg, - abortSignal: controller.signal, - stopWhen: stepCountIs(stepLimit), - }).finally(() => clearTimeout(timeoutId)); - - const content: ChatContent = result.steps.flatMap((step) => { - const blocks: ChatContent = []; - if (step.text) { - blocks.push({ type: "text", text: step.text }); - } - const outById = new Map(step.toolResults.map((r) => [r.toolCallId, r.output as Json])); - for (const call of step.toolCalls) { - blocks.push({ - type: "tool-call", - toolName: call.toolName, - toolCallId: call.toolCallId, - args: call.input as Json, - argsText: JSON.stringify(call.input), - result: outById.get(call.toolCallId) ?? null, - }); - } - return blocks; - }); - - let responseConversationId: string | undefined; - if (body.mcpCallMetadata != null) { - const correlationId = crypto.randomUUID(); - const conversationId = body.mcpCallMetadata.conversationId ?? crypto.randomUUID(); - responseConversationId = conversationId; - const firstUserMessage = messages.find(m => m.role === "user"); - const question = typeof firstUserMessage?.content === "string" - ? firstUserMessage.content - : JSON.stringify(firstUserMessage?.content ?? ""); - - const innerToolCallsJson = JSON.stringify(content.filter(b => b.type === "tool-call")); + const ctx: ModeContext = { model, cachedMessages, toolsArg, stepLimit, common, startedAt }; + const extras = { + messages, + mcpCallMetadata: body.mcpCallMetadata ?? undefined, + correlationId, + conversationIdForLog, + }; - const logPromise = logMcpCall({ - correlationId, - toolName: body.mcpCallMetadata.toolName, - reason: body.mcpCallMetadata.reason, - userPrompt: body.mcpCallMetadata.userPrompt, - conversationId, - question, - response: result.text, - stepCount: result.steps.length, - innerToolCallsJson, - durationMs: BigInt(Date.now() - startedAt), - modelId: String(model.modelId), - errorMessage: undefined, - }); - runAsynchronouslyAndWaitUntil(logPromise); - - runAsynchronouslyAndWaitUntil(reviewMcpCall({ - logPromise, - correlationId, - question, - reason: body.mcpCallMetadata.reason, - response: result.text, - })); - } - - return { - statusCode: 200, - bodyType: "json" as const, - body: { - content, - finalText: result.text, - conversationId: responseConversationId ?? null, - }, - }; + if (mode === "stream") { + return handleStreamMode({ ...ctx, ...extras }); } + return await handleGenerateMode({ ...ctx, ...extras }); }, }); diff --git a/apps/backend/src/app/api/latest/integrations/ai-proxy/[[...path]]/route.ts b/apps/backend/src/app/api/latest/integrations/ai-proxy/[[...path]]/route.ts index a45d49cc6e..426070aee9 100644 --- a/apps/backend/src/app/api/latest/integrations/ai-proxy/[[...path]]/route.ts +++ b/apps/backend/src/app/api/latest/integrations/ai-proxy/[[...path]]/route.ts @@ -1,97 +1,60 @@ -import { ALLOWED_MODEL_IDS } from "@/lib/ai/models"; -import { preprocessProxyBody } from "@/private"; +import { observeAndLog, sanitizeBody } from "@/lib/ai/ai-proxy-handlers"; import { handleApiRequest } from "@/route-handlers/smart-route-handler"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { NextRequest } from "next/server"; const OPENROUTER_BASE_URL = "https://openrouter.ai/api"; const PRODUCTION_PROXY_BASE_URL = "https://api.stack-auth.com/api/latest/integrations/ai-proxy"; -const OPENROUTER_DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"; - -function sanitizeBody(raw: ArrayBuffer): Uint8Array { - const text = new TextDecoder().decode(raw); - let parsed; - try { - parsed = JSON.parse(text); - } catch { - throw new StatusError(400, "Request body must be valid JSON"); - } - - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { - throw new StatusError(400, "Request body must be a JSON object"); - } - - if (!parsed.model || !ALLOWED_MODEL_IDS.has(parsed.model)) { - parsed.model = OPENROUTER_DEFAULT_MODEL; - } - - // OpenRouter limits metadata.user_id to 128 characters - if (parsed.metadata?.user_id && parsed.metadata.user_id.length > 128) { - parsed.metadata.user_id = parsed.metadata.user_id.slice(0, 128); - } - - parsed = preprocessProxyBody({ - parsedBody: parsed, - }); - - return new TextEncoder().encode(JSON.stringify(parsed)); -} async function proxyToOpenRouter(req: NextRequest, options: { params: Promise<{ path?: string[] }> }) { const apiKey = getEnvVariable("STACK_OPENROUTER_API_KEY"); const params = await options.params; const subpath = params.path?.join("/") ?? ""; - const contentType = req.headers.get("Content-Type"); - const body = req.method !== "GET" && req.method !== "HEAD" - ? Buffer.from(sanitizeBody(await req.arrayBuffer())) + const sanitized = req.method !== "GET" && req.method !== "HEAD" + ? sanitizeBody(await req.arrayBuffer()) : undefined; + const body = sanitized ? Buffer.from(sanitized.bytes) : undefined; + const callerApiKey = req.headers.get("x-api-key"); + const shouldLog = sanitized != null && callerApiKey != null && callerApiKey.startsWith("stack-auth-"); + const correlationId = crypto.randomUUID(); + const startedAt = performance.now(); + + const targetUrl = apiKey === "FORWARD_TO_PRODUCTION" + ? `${PRODUCTION_PROXY_BASE_URL}/${subpath}${req.nextUrl.search}` + : `${OPENROUTER_BASE_URL}/${subpath}${req.nextUrl.search}`; + const forwardHeaders: Record = apiKey === "FORWARD_TO_PRODUCTION" + ? {} + : { + "Authorization": `Bearer ${apiKey}`, + "anthropic-version": "2023-06-01", + }; + if (body) forwardHeaders["Content-Type"] = "application/json"; + + const response = await fetch(targetUrl, { method: req.method, headers: forwardHeaders, body }); + + const responseHeaders = { + "Content-Type": response.headers.get("Content-Type") ?? "application/json", + "Cache-Control": "no-store", + }; - if (apiKey === "FORWARD_TO_PRODUCTION") { - const targetUrl = `${PRODUCTION_PROXY_BASE_URL}/${subpath}${req.nextUrl.search}`; - const headers: Record = {}; - if (contentType) { - headers["Content-Type"] = contentType; - } + const passthrough = () => new Response(response.body, { status: response.status, headers: responseHeaders }); - const response = await fetch(targetUrl, { - method: req.method, - headers, - body, - }); - - return new Response(response.body, { - status: response.status, - headers: { - "Content-Type": response.headers.get("Content-Type") ?? "application/json", - "Cache-Control": "no-cache", - }, + if (!shouldLog) return passthrough(); + try { + return await observeAndLog({ + response, + sanitizedBody: sanitized!, + callerApiKey, + correlationId, + startedAt, + responseHeaders, }); + } catch (e) { + captureError("ai-proxy-log-pipeline", e); + return passthrough(); } - - const targetUrl = `${OPENROUTER_BASE_URL}/${subpath}${req.nextUrl.search}`; - const headers: Record = { - "Authorization": `Bearer ${apiKey}`, - "anthropic-version": "2023-06-01", - }; - if (contentType) { - headers["Content-Type"] = contentType; - } - - const response = await fetch(targetUrl, { - method: req.method, - headers, - body, - }); - - return new Response(response.body, { - status: response.status, - headers: { - "Content-Type": response.headers.get("Content-Type") ?? "application/json", - "Cache-Control": "no-cache", - }, - }); } export const GET = handleApiRequest(proxyToOpenRouter); diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts index fed36bcc9f..c97f9e8572 100644 --- a/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts +++ b/apps/backend/src/app/api/latest/internal/mcp-review/add-manual/route.ts @@ -1,8 +1,8 @@ -import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; -import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export const POST = createSmartRouteHandler({ metadata: { hidden: true }, @@ -28,23 +28,16 @@ export const POST = createSmartRouteHandler({ }), handler: async ({ auth, body }) => { const user = auth.user; - if (getNodeEnvironment() !== "development") { - const metadata = user.client_read_only_metadata; - if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { - throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); - } - } - - const conn = await getConnectionOrThrow(); + assertIsAiChatReviewer(user); const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.addManualQa({ + await callReducerStrict("add_manual_qa", [ token, - question: body.question, - answer: body.answer, - publish: body.publish, - reviewedBy: user.display_name ?? user.primary_email ?? user.id, - }); + body.question, + body.answer, + body.publish, + user.display_name ?? user.primary_email ?? user.id, + ]); return { statusCode: 200, diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts index e3126c65d0..b36f4433bc 100644 --- a/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts +++ b/apps/backend/src/app/api/latest/internal/mcp-review/delete/route.ts @@ -1,8 +1,8 @@ -import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; -import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export const POST = createSmartRouteHandler({ metadata: { hidden: true }, @@ -13,7 +13,7 @@ export const POST = createSmartRouteHandler({ project: adaptSchema, }).defined(), body: yupObject({ - correlationId: yupString().defined(), + qaId: yupString().matches(/^\d+$/, "qaId must be a non-negative decimal integer").defined(), }).defined(), method: yupString().oneOf(["POST"]).defined(), }), @@ -25,20 +25,13 @@ export const POST = createSmartRouteHandler({ }).defined(), }), handler: async ({ auth, body }) => { - if (getNodeEnvironment() !== "development") { - const metadata = auth.user.client_read_only_metadata; - if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { - throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); - } - } - - const conn = await getConnectionOrThrow(); + assertIsAiChatReviewer(auth.user); const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.deleteQaEntry({ + await callReducerStrict("delete_qa_entry", [ token, - correlationId: body.correlationId, - }); + BigInt(body.qaId), + ]); return { statusCode: 200, diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts index 5500fe15ba..624fe2c9ba 100644 --- a/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts +++ b/apps/backend/src/app/api/latest/internal/mcp-review/mark-reviewed/route.ts @@ -1,8 +1,8 @@ -import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; -import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export const POST = createSmartRouteHandler({ metadata: { hidden: true }, @@ -26,21 +26,14 @@ export const POST = createSmartRouteHandler({ }), handler: async ({ auth, body }) => { const user = auth.user; - if (getNodeEnvironment() !== "development") { - const metadata = user.client_read_only_metadata; - if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { - throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); - } - } - - const conn = await getConnectionOrThrow(); + assertIsAiChatReviewer(user); const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.markHumanReviewed({ + await callReducerStrict("mark_human_reviewed", [ token, - correlationId: body.correlationId, - reviewedBy: user.display_name ?? user.primary_email ?? user.id, - }); + body.correlationId, + user.display_name ?? user.primary_email ?? user.id, + ]); return { statusCode: 200, diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/unmark-reviewed/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/unmark-reviewed/route.ts new file mode 100644 index 0000000000..37d61ae00c --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/unmark-reviewed/route.ts @@ -0,0 +1,42 @@ +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + correlationId: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + assertIsAiChatReviewer(auth.user); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await callReducerStrict("unmark_human_reviewed", [ + token, + body.correlationId, + ]); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts index dfb40a7859..10bf2d17df 100644 --- a/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts +++ b/apps/backend/src/app/api/latest/internal/mcp-review/update-correction/route.ts @@ -1,8 +1,8 @@ -import { getConnectionOrThrow } from "@/lib/ai/mcp-logger"; +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; -import { getEnvVariable, getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; -import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; export const POST = createSmartRouteHandler({ metadata: { hidden: true }, @@ -29,24 +29,20 @@ export const POST = createSmartRouteHandler({ }), handler: async ({ auth, body }) => { const user = auth.user; - if (getNodeEnvironment() !== "development") { - const metadata = user.client_read_only_metadata; - if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { - throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); - } - } - - const conn = await getConnectionOrThrow(); + assertIsAiChatReviewer(user); const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.updateHumanCorrection({ + const reviewer = user.display_name ?? user.primary_email ?? user.id; + + await callReducerStrict("upsert_qa_from_call", [ token, - correlationId: body.correlationId, - correctedQuestion: body.correctedQuestion, - correctedAnswer: body.correctedAnswer, - publish: body.publish, - reviewedBy: user.display_name ?? user.primary_email ?? user.id, - }); + body.correlationId, + body.correctedQuestion, + body.correctedAnswer, + body.publish, + reviewer, + ]); + await callReducerStrict("mark_human_reviewed", [token, body.correlationId, reviewer]); return { statusCode: 200, diff --git a/apps/backend/src/app/api/latest/internal/mcp-review/update-qa-entry/route.ts b/apps/backend/src/app/api/latest/internal/mcp-review/update-qa-entry/route.ts new file mode 100644 index 0000000000..38fb8ee7db --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/mcp-review/update-qa-entry/route.ts @@ -0,0 +1,52 @@ +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + qaId: yupString().matches(/^\d+$/, "qaId must be a non-negative decimal integer").defined(), + question: yupString().defined(), + answer: yupString().defined(), + publish: yupBoolean().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + const user = auth.user; + assertIsAiChatReviewer(user); + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + const editor = user.display_name ?? user.primary_email ?? user.id; + const qaId = BigInt(body.qaId); + await callReducerStrict("update_qa_entry_with_publish", [ + token, + qaId, + body.question, + body.answer, + body.publish, + editor, + ]); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/internal/spacetimedb-enroll-reviewer/route.ts b/apps/backend/src/app/api/latest/internal/spacetimedb-enroll-reviewer/route.ts new file mode 100644 index 0000000000..bb2ae0ecb6 --- /dev/null +++ b/apps/backend/src/app/api/latest/internal/spacetimedb-enroll-reviewer/route.ts @@ -0,0 +1,49 @@ +import { callReducerStrict } from "@/lib/ai/spacetimedb-client"; +import { assertIsAiChatReviewer } from "@/lib/ai/qa/reviewer-auth"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { adaptSchema, yupBoolean, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const POST = createSmartRouteHandler({ + metadata: { hidden: true }, + request: yupObject({ + auth: yupObject({ + type: adaptSchema, + user: adaptSchema.defined(), + project: adaptSchema, + }).defined(), + body: yupObject({ + identity: yupString().defined(), + }).defined(), + method: yupString().oneOf(["POST"]).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + success: yupBoolean().defined(), + }).defined(), + }), + handler: async ({ auth, body }) => { + const user = auth.user; + assertIsAiChatReviewer(user); + if (!/^[0-9a-fA-F]{64}$/.test(body.identity)) { + throw new StatusError(StatusError.BadRequest, "Invalid identity."); + } + + const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); + await callReducerStrict("add_operator", [ + token, + [`0x${body.identity}`], + user.id, + user.display_name ?? user.primary_email ?? user.id, + ]); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { success: true }, + }; + }, +}); diff --git a/apps/backend/src/lib/ai/ai-proxy-handlers.ts b/apps/backend/src/lib/ai/ai-proxy-handlers.ts new file mode 100644 index 0000000000..d2b985de00 --- /dev/null +++ b/apps/backend/src/lib/ai/ai-proxy-handlers.ts @@ -0,0 +1,92 @@ +import { buildProxyLogRow, scheduleProxyLog } from "@/lib/ai/loggers/ai-proxy-logger"; +import { ALLOWED_MODEL_IDS } from "@/lib/ai/models"; +import { extractOpenRouterUsage, scanSseForUsage, type UsageFields } from "@/lib/ai/openrouter-usage"; +import { preprocessProxyBody } from "@/private"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; +import { captureError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export const OPENROUTER_DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"; + +export type SanitizedBody = { + parsed: { model: string } & Record, + bytes: Uint8Array, +}; + +export function sanitizeBody(raw: ArrayBuffer): SanitizedBody { + const text = new TextDecoder().decode(raw); + let parsed; + try { + parsed = JSON.parse(text); + } catch { + throw new StatusError(400, "Request body must be valid JSON"); + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new StatusError(400, "Request body must be a JSON object"); + } + + if (!parsed.model || !ALLOWED_MODEL_IDS.has(parsed.model)) { + parsed.model = OPENROUTER_DEFAULT_MODEL; + } + + if (parsed.metadata?.user_id && parsed.metadata.user_id.length > 128) { + parsed.metadata.user_id = parsed.metadata.user_id.slice(0, 128); + } + + const processed = preprocessProxyBody({ parsedBody: parsed }) as { model: string } & Record; + return { parsed: processed, bytes: new TextEncoder().encode(JSON.stringify(processed)) }; +} + +export async function observeAndLog(args: { + response: Response, + sanitizedBody: SanitizedBody, + callerApiKey: string, + correlationId: string, + startedAt: number, + responseHeaders: Record, +}): Promise { + const { response, sanitizedBody, callerApiKey, correlationId, startedAt, responseHeaders } = args; + const isStreaming = sanitizedBody.parsed.stream === true; + + if (isStreaming && response.body) { + const [clientStream, observerStream] = response.body.tee(); + runAsynchronouslyAndWaitUntil((async () => { + let usage: UsageFields = {}; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 120_000); + try { + usage = (await scanSseForUsage(observerStream, controller.signal)) ?? {}; + } catch (err) { + captureError("ai-proxy-scan-sse", err); + } finally { + clearTimeout(timeoutId); + } + scheduleProxyLog(buildProxyLogRow({ + correlationId, + parsed: sanitizedBody.parsed, + apiKey: callerApiKey, + durationMs: BigInt(Math.round(performance.now() - startedAt)), + responseStatus: response.status, + usage, + })); + })()); + return new Response(clientStream, { status: response.status, headers: responseHeaders }); + } + + const bodyBytes = await response.arrayBuffer(); + let parsedBody: unknown; + try { + parsedBody = JSON.parse(new TextDecoder().decode(bodyBytes)); + } catch { + parsedBody = undefined; + } + scheduleProxyLog(buildProxyLogRow({ + correlationId, + parsed: sanitizedBody.parsed, + apiKey: callerApiKey, + durationMs: BigInt(Math.round(performance.now() - startedAt)), + responseStatus: response.status, + usage: extractOpenRouterUsage(parsedBody), + })); + return new Response(bodyBytes, { status: response.status, headers: responseHeaders }); +} diff --git a/apps/backend/src/lib/ai/ai-query-handlers.ts b/apps/backend/src/lib/ai/ai-query-handlers.ts new file mode 100644 index 0000000000..8b9916d03b --- /dev/null +++ b/apps/backend/src/lib/ai/ai-query-handlers.ts @@ -0,0 +1,169 @@ +import { logAiQueryFailure, logAiQuerySuccess } from "@/lib/ai/loggers/ai-query-logger"; +import { logIfMcpToolCall } from "@/lib/ai/loggers/mcp-call-logger"; +import type { ContentBlock, McpCallMetadata, MessageLike, ModeContext } from "@/lib/ai/types"; +import { listManagedProjectIds } from "@/lib/projects"; +import type { SmartRequestAuth } from "@/route-handlers/smart-request"; +import { captureError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; +import { Json } from "@stackframe/stack-shared/dist/utils/json"; +import { generateText, stepCountIs, streamText, type StepResult, type ToolSet } from "ai"; + +export const USER_FACING_ERROR_MESSAGE = "The AI service is temporarily unavailable. Please try again later."; + +export const OPENROUTER_PROVIDER_OPTIONS = { + usage: { include: true }, + stream_options: { include_usage: true }, +} as const; + +export async function assertProjectAccess(projectId: string, auth: SmartRequestAuth | null): Promise { + if (auth == null || auth.project.id !== "internal" || auth.user == null) { + throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); + } + const managedProjectIds = await listManagedProjectIds(auth.user); + if (!managedProjectIds.includes(projectId)) { + throw new StatusError(StatusError.Forbidden, "You do not have access to this project"); + } +} + +export function handleStreamMode(ctx: ModeContext & { + messages: ReadonlyArray, + mcpCallMetadata: McpCallMetadata | undefined, + correlationId: string, + conversationIdForLog: string | undefined, +}) { + const { model, cachedMessages, toolsArg, stepLimit, common, startedAt, messages, mcpCallMetadata, correlationId, conversationIdForLog } = ctx; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 120_000); + const completedSteps: StepResult[] = []; + const result = streamText({ + model, + messages: cachedMessages, + tools: toolsArg, + abortSignal: controller.signal, + stopWhen: stepCountIs(stepLimit), + providerOptions: { openrouter: OPENROUTER_PROVIDER_OPTIONS }, + onStepFinish: (step) => { completedSteps.push(step); }, + onFinish: ({ text, steps, usage, providerMetadata, response }) => { + clearTimeout(timeoutId); + logAiQuerySuccess({ + common, + startedAt, + steps, + text, + usage, + providerMetadata, + openrouterGenerationId: response.id, + }); + logIfMcpToolCall({ + mcpCallMetadata, + conversationIdForLog, + correlationId, + messages, + steps, + text, + startedAt, + modelId: String(model.modelId), + }); + }, + onError: ({ error }) => { + clearTimeout(timeoutId); + logAiQueryFailure({ common, startedAt, err: error, partialSteps: completedSteps }); + }, + onAbort: () => { + clearTimeout(timeoutId); + logAiQueryFailure({ + common, + startedAt, + err: new Error("Stream aborted (client disconnect or timeout)"), + partialSteps: completedSteps, + }); + }, + }); + return { + statusCode: 200, + bodyType: "response" as const, + body: result.toUIMessageStreamResponse({ + onError: (err) => { + captureError("ai-query-stream-writer", err); + return USER_FACING_ERROR_MESSAGE; + }, + }), + }; +} + +export async function handleGenerateMode(ctx: ModeContext & { + messages: ReadonlyArray, + mcpCallMetadata: McpCallMetadata | undefined, + correlationId: string, + conversationIdForLog: string | undefined, +}) { + const { model, cachedMessages, toolsArg, stepLimit, common, startedAt, messages, mcpCallMetadata, correlationId, conversationIdForLog } = ctx; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 120_000); + const completedSteps: StepResult[] = []; + let result: Awaited>; + try { + result = await generateText({ + model, + messages: cachedMessages, + tools: toolsArg, + abortSignal: controller.signal, + stopWhen: stepCountIs(stepLimit), + providerOptions: { openrouter: OPENROUTER_PROVIDER_OPTIONS }, + onStepFinish: (step) => { completedSteps.push(step); }, + }).finally(() => clearTimeout(timeoutId)); + } catch (err) { + logAiQueryFailure({ common, startedAt, err, partialSteps: completedSteps }); + throw new StatusError(StatusError.BadGateway, USER_FACING_ERROR_MESSAGE); + } + + const contentBlocks: ContentBlock[] = []; + for (const step of result.steps) { + if (step.text) { + contentBlocks.push({ type: "text", text: step.text }); + } + const resultsByCallId = new Map(step.toolResults.map(r => [r.toolCallId, r])); + for (const toolCall of step.toolCalls) { + const toolResult = resultsByCallId.get(toolCall.toolCallId); + contentBlocks.push({ + type: "tool-call", + toolName: toolCall.toolName, + toolCallId: toolCall.toolCallId, + args: toolCall.input, + argsText: JSON.stringify(toolCall.input), + result: (toolResult?.output ?? null) as Json, + }); + } + } + + logAiQuerySuccess({ + common, + startedAt, + steps: result.steps, + text: result.text, + usage: result.usage, + providerMetadata: result.providerMetadata, + openrouterGenerationId: result.response.id, + }); + + const responseConversationId = mcpCallMetadata != null ? conversationIdForLog : undefined; + logIfMcpToolCall({ + mcpCallMetadata, + conversationIdForLog, + correlationId, + messages, + steps: result.steps, + text: result.text, + startedAt, + modelId: String(model.modelId), + }); + + return { + statusCode: 200, + bodyType: "json" as const, + body: { + content: contentBlocks, + finalText: result.text, + conversationId: responseConversationId ?? null, + }, + }; +} diff --git a/apps/backend/src/lib/ai/loggers/ai-proxy-logger.ts b/apps/backend/src/lib/ai/loggers/ai-proxy-logger.ts new file mode 100644 index 0000000000..3a51767511 --- /dev/null +++ b/apps/backend/src/lib/ai/loggers/ai-proxy-logger.ts @@ -0,0 +1,72 @@ +import { logAiQuery, type AiQueryLogEntry } from "@/lib/ai/loggers/ai-query-logger"; +import { refineGenerationCost, type UsageFields } from "@/lib/ai/openrouter-usage"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; + +export type ProxyLogFields = { + correlationId: string, + parsed: { model: string } & Record, + apiKey: string, + durationMs: bigint, + responseStatus: number, + usage?: UsageFields, +}; + +export function buildProxyLogRow(fields: ProxyLogFields): AiQueryLogEntry { + const { parsed, apiKey, durationMs, responseStatus, usage, correlationId } = fields; + const tools = Array.isArray(parsed.tools) ? parsed.tools : []; + const toolNames = tools + .map(t => (t && typeof t === "object" && "name" in t) ? (t as { name: unknown }).name : null) + .filter((n): n is string => typeof n === "string"); + const rawMessages = Array.isArray(parsed.messages) ? parsed.messages : []; + const messages = typeof parsed.system === "string" && parsed.system.length > 0 + ? [{ role: "system", content: parsed.system }, ...rawMessages] + : rawMessages; + return { + correlationId, + mode: parsed.stream === true ? "stream" : "generate", + systemPromptId: apiKey === "stack-auth-proxy" ? "stack-cli" : apiKey, + quality: "unknown", + speed: "unknown", + modelId: parsed.model, + isAuthenticated: false, + projectId: undefined, + userId: undefined, + requestedToolsJson: JSON.stringify(toolNames), + messagesJson: JSON.stringify(messages), + stepsJson: "[]", + finalText: "", + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + cachedInputTokens: usage?.cachedInputTokens, + cacheCreationTokens: usage?.cacheCreationTokens, + costUsd: usage?.costUsd, + cacheDiscountUsd: undefined, + openrouterGenerationId: usage?.generationId, + stepCount: 0, + durationMs, + errorMessage: responseStatus >= 400 ? `upstream ${responseStatus}` : undefined, + conversationId: undefined, + }; +} + +export function scheduleProxyLog(row: AiQueryLogEntry): void { + try { + const safe = (async () => { + try { + await logAiQuery(row); + } catch (e) { + captureError("ai-proxy-log-async", e); + } + })(); + runAsynchronouslyAndWaitUntil(safe); + if (row.openrouterGenerationId != null) { + runAsynchronouslyAndWaitUntil(refineGenerationCost({ + generationId: row.openrouterGenerationId, + correlationId: row.correlationId, + })); + } + } catch (e) { + captureError("ai-proxy-log-sync", e); + } +} diff --git a/apps/backend/src/lib/ai/loggers/ai-query-logger.ts b/apps/backend/src/lib/ai/loggers/ai-query-logger.ts new file mode 100644 index 0000000000..5cb08a051b --- /dev/null +++ b/apps/backend/src/lib/ai/loggers/ai-query-logger.ts @@ -0,0 +1,196 @@ +import { extractCachedTokens, extractCostFromUsage, extractOpenRouterCost, refineGenerationCost } from "@/lib/ai/openrouter-usage"; +import type { CommonLogFields } from "@/lib/ai/types"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError, StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { type LanguageModelUsage, type StepResult, type ToolSet } from "ai"; +import { callReducer, opt } from "../spacetimedb-client"; + +const MAX_TOOL_RESULT_CHARS = 50_000; + +function sanitizeOptionalNumber(name: string, n: number | undefined): number | undefined { + if (n == null) return undefined; + if (!Number.isFinite(n)) { + captureError("ai-query-logger", new StackAssertionError(`Invalid ${name}: ${n}`)); + return undefined; + } + return n; +} + +function sanitizeRequiredNumber(name: string, n: number): number { + if (!Number.isFinite(n)) { + captureError("ai-query-logger", new StackAssertionError(`Invalid ${name}: ${n}`)); + return 0; + } + return n; +} + +function truncateLargeToolResult(toolName: string, output: unknown): unknown { + const serialized = JSON.stringify(output); + if (serialized.length <= MAX_TOOL_RESULT_CHARS) return output; + captureError( + "ai-query-tool-result-truncated", + new StackAssertionError( + `Tool ${toolName} returned ${serialized.length} chars (limit ${MAX_TOOL_RESULT_CHARS}); truncating in stepsJson log.` + ) + ); + return { + _truncated: true, + originalSize: serialized.length, + preview: serialized.slice(0, 10000), + }; +} + +function formatErrorForLog(err: unknown): string { + if (!(err instanceof Error)) return String(err); + const code = (err as { code?: unknown }).code; + const codePart = typeof code === "string" || typeof code === "number" ? ` [code=${code}]` : ""; + const base = err.stack ?? `${err.name}: ${err.message}`; + return `${base}${codePart}`; +} + +export type AiQueryLogEntry = { + correlationId: string, + mode: string, + systemPromptId: string, + quality: string, + speed: string, + modelId: string, + isAuthenticated: boolean, + projectId: string | undefined, + userId: string | undefined, + requestedToolsJson: string, + messagesJson: string, + stepsJson: string, + finalText: string, + inputTokens: number | undefined, + outputTokens: number | undefined, + cachedInputTokens: number | undefined, + cacheCreationTokens: number | undefined, + costUsd: number | undefined, + cacheDiscountUsd: number | undefined, + openrouterGenerationId: string | undefined, + stepCount: number, + durationMs: bigint, + errorMessage: string | undefined, + conversationId: string | undefined, +}; + +export async function logAiQuery(entry: AiQueryLogEntry): Promise { + const logToken = getEnvVariable("STACK_MCP_LOG_TOKEN", ""); + await callReducer("log_ai_query", [ + logToken, + entry.correlationId, + entry.mode, + entry.systemPromptId, + entry.quality, + entry.speed, + entry.modelId, + entry.isAuthenticated, + opt(entry.projectId), + opt(entry.userId), + entry.requestedToolsJson, + entry.messagesJson, + entry.stepsJson, + entry.finalText, + opt(sanitizeOptionalNumber("inputTokens", entry.inputTokens)), + opt(sanitizeOptionalNumber("outputTokens", entry.outputTokens)), + opt(sanitizeOptionalNumber("cachedInputTokens", entry.cachedInputTokens)), + opt(sanitizeOptionalNumber("cacheCreationTokens", entry.cacheCreationTokens)), + opt(sanitizeOptionalNumber("costUsd", entry.costUsd)), + opt(sanitizeOptionalNumber("cacheDiscountUsd", entry.cacheDiscountUsd)), + opt(entry.openrouterGenerationId), + sanitizeRequiredNumber("stepCount", entry.stepCount), + entry.durationMs, + opt(entry.errorMessage), + opt(entry.conversationId), + ]); +} + +function serializeSteps(steps: ReadonlyArray>): string { + try { + return JSON.stringify(steps.map((step, i) => ({ + step: i, + text: step.text || undefined, + toolCalls: step.toolCalls.map(tc => ({ + toolName: tc.toolName, + toolCallId: tc.toolCallId, + args: tc.input, + })), + toolResults: step.toolResults.map(tr => ({ + toolName: tr.toolName, + toolCallId: tr.toolCallId, + result: truncateLargeToolResult(tr.toolName, tr.output), + })), + }))); + } catch (e) { + captureError("ai-query-steps-serialize", e); + return JSON.stringify({ _serializationFailed: true, stepCount: steps.length }); + } +} + +export function logAiQuerySuccess(args: { + common: CommonLogFields, + startedAt: number, + steps: ReadonlyArray>, + text: string, + usage: LanguageModelUsage, + providerMetadata: unknown, + openrouterGenerationId: string | undefined, +}): void { + const { common, startedAt, steps, text, usage, providerMetadata, openrouterGenerationId } = args; + // Build the row inside the async task so any throw (serialization, + // metadata extraction, etc.) is contained by the async boundary instead + // of bubbling up into the user-facing success path. + runAsynchronouslyAndWaitUntil(async () => { + const rawCost = extractCostFromUsage(usage); + await logAiQuery({ + ...common, + stepsJson: serializeSteps(steps), + finalText: text, + inputTokens: usage.inputTokens ?? undefined, + outputTokens: usage.outputTokens ?? undefined, + cachedInputTokens: extractCachedTokens(providerMetadata), + cacheCreationTokens: usage.inputTokenDetails.cacheWriteTokens ?? undefined, + costUsd: rawCost.costUsd ?? extractOpenRouterCost(providerMetadata), + cacheDiscountUsd: undefined, // backfilled by refineGenerationCost below + openrouterGenerationId, + stepCount: steps.length, + durationMs: BigInt(Math.round(performance.now() - startedAt)), + errorMessage: undefined, + }); + }); + if (openrouterGenerationId != null) { + runAsynchronouslyAndWaitUntil(refineGenerationCost({ + generationId: openrouterGenerationId, + correlationId: common.correlationId, + })); + } +} + +export function logAiQueryFailure(args: { + common: CommonLogFields, + startedAt: number, + err: unknown, + partialSteps?: ReadonlyArray>, +}): void { + const { common, startedAt, err, partialSteps } = args; + captureError("ai-query-upstream", err); + runAsynchronouslyAndWaitUntil(async () => { + await logAiQuery({ + ...common, + stepsJson: partialSteps && partialSteps.length > 0 ? serializeSteps(partialSteps) : "[]", + finalText: "", + inputTokens: undefined, + outputTokens: undefined, + cachedInputTokens: undefined, + cacheCreationTokens: undefined, + costUsd: undefined, + cacheDiscountUsd: undefined, + openrouterGenerationId: undefined, + stepCount: partialSteps?.length ?? 0, + durationMs: BigInt(Math.round(performance.now() - startedAt)), + errorMessage: formatErrorForLog(err), + }); + }); +} diff --git a/apps/backend/src/lib/ai/loggers/mcp-call-logger.ts b/apps/backend/src/lib/ai/loggers/mcp-call-logger.ts new file mode 100644 index 0000000000..fccb3176cc --- /dev/null +++ b/apps/backend/src/lib/ai/loggers/mcp-call-logger.ts @@ -0,0 +1,100 @@ +import { reviewMcpCall } from "@/lib/ai/qa/qa-reviewer"; +import type { McpCallMetadata, MessageLike } from "@/lib/ai/types"; +import { runAsynchronouslyAndWaitUntil } from "@/utils/background-tasks"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { Json } from "@stackframe/stack-shared/dist/utils/json"; +import { type StepResult, type ToolSet } from "ai"; +import { callReducer, opt } from "../spacetimedb-client"; + +export type McpLogEntry = { + correlationId: string, + toolName: string, + reason: string, + userPrompt: string, + conversationId: string | undefined, + question: string, + response: string, + stepCount: number, + innerToolCallsJson: string, + durationMs: bigint, + modelId: string, + errorMessage: string | undefined, +}; + +export async function logMcpCall(entry: McpLogEntry): Promise { + const logToken = getEnvVariable("STACK_MCP_LOG_TOKEN", ""); + await callReducer("log_mcp_call", [ + logToken, + entry.correlationId, + opt(entry.conversationId), + entry.toolName, + entry.reason, + entry.userPrompt, + entry.question, + entry.response, + entry.stepCount, + entry.innerToolCallsJson, + entry.durationMs, + entry.modelId, + opt(entry.errorMessage), + ]); +} + +function buildInnerToolCallsJson(steps: ReadonlyArray>): string { + const items: Json[] = []; + for (const step of steps) { + const resultsByCallId = new Map(step.toolResults.map(r => [r.toolCallId, r])); + for (const tc of step.toolCalls) { + items.push({ + type: "tool-call", + toolName: tc.toolName, + toolCallId: tc.toolCallId, + args: tc.input as Json, + argsText: JSON.stringify(tc.input), + result: (resultsByCallId.get(tc.toolCallId)?.output ?? null) as Json, + }); + } + } + return JSON.stringify(items); +} + +export function logIfMcpToolCall(args: { + mcpCallMetadata: McpCallMetadata | undefined, + conversationIdForLog: string | undefined, + correlationId: string, + messages: ReadonlyArray, + steps: ReadonlyArray>, + text: string, + startedAt: number, + modelId: string, +}): void { + const { mcpCallMetadata, conversationIdForLog, correlationId, messages, steps, text, startedAt, modelId } = args; + if (mcpCallMetadata == null || conversationIdForLog == null) return; + const lastUserMessage = messages.findLast(m => m.role === "user"); + const question = typeof lastUserMessage?.content === "string" + ? lastUserMessage.content + : JSON.stringify(lastUserMessage?.content ?? ""); + const innerToolCallsJson = buildInnerToolCallsJson(steps); + const logPromise = logMcpCall({ + correlationId, + toolName: mcpCallMetadata.toolName, + reason: mcpCallMetadata.reason, + userPrompt: mcpCallMetadata.userPrompt, + conversationId: conversationIdForLog, + question, + response: text, + stepCount: steps.length, + innerToolCallsJson, + durationMs: BigInt(Math.round(performance.now() - startedAt)), + modelId, + errorMessage: undefined, + }); + runAsynchronouslyAndWaitUntil(logPromise); + runAsynchronouslyAndWaitUntil(reviewMcpCall({ + logPromise, + correlationId, + question, + reason: mcpCallMetadata.reason, + response: text, + })); +} diff --git a/apps/backend/src/lib/ai/mcp-logger.ts b/apps/backend/src/lib/ai/mcp-logger.ts deleted file mode 100644 index f8a9a3d045..0000000000 --- a/apps/backend/src/lib/ai/mcp-logger.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; -import { StackAssertionError, captureError } from "@stackframe/stack-shared/dist/utils/errors"; -import { DbConnection } from "./spacetimedb-bindings"; -import type { LogMcpCallParams } from "./spacetimedb-bindings/types/reducers"; - -export type McpLogEntry = Omit; - -let connectionPromise: Promise | null = null; - -export async function getConnection(): Promise { - const uri = getEnvVariable("STACK_SPACETIMEDB_URI", ""); - if (!uri) { - return null; - } - - if (!connectionPromise) { - connectionPromise = new Promise((resolve, reject) => { - DbConnection.builder() - .withUri(uri) - .withDatabaseName(getEnvVariable("STACK_SPACETIMEDB_DB_NAME")) - .onConnect((connInstance) => { - connInstance.subscriptionBuilder() - .onApplied(() => { - resolve(connInstance); - }) - .subscribe("SELECT * FROM mcp_call_log"); - }) - .onConnectError((_: unknown, err: Error) => { - captureError("mcp-logger", err); - connectionPromise = null; - reject(err); - }) - .build(); - }); - } - - return await connectionPromise; -} - -export async function getConnectionOrThrow(): Promise { - const conn = await getConnection(); - if (!conn) { - throw new StackAssertionError("SpacetimeDB connection unavailable"); - } - return conn; -} - -export async function logMcpCall(entry: McpLogEntry): Promise { - const conn = await getConnection(); - if (!conn) { - return; - } - - const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.logMcpCall({ - token, - ...entry, - }); -} diff --git a/apps/backend/src/lib/ai/openrouter-usage.ts b/apps/backend/src/lib/ai/openrouter-usage.ts new file mode 100644 index 0000000000..df694c3331 --- /dev/null +++ b/apps/backend/src/lib/ai/openrouter-usage.ts @@ -0,0 +1,190 @@ +import { callReducer, opt } from "@/lib/ai/spacetimedb-client"; +import type { OpenRouterUsageAccounting } from "@openrouter/ai-sdk-provider"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import type { LanguageModelUsage } from "ai"; + +export type UsageFields = { + inputTokens?: number, + outputTokens?: number, + cachedInputTokens?: number, + cacheCreationTokens?: number, + costUsd?: number, + generationId?: string, +}; + +type ProviderMetadata = { openrouter?: { usage?: OpenRouterUsageAccounting } }; + + +export function extractCostFromUsage(usage: LanguageModelUsage): { + costUsd?: number, +} { + const raw = usage.raw as RawUsage | undefined; + if (raw == null) return {}; + return { costUsd: raw.cost }; +} + +export function extractOpenRouterCost(meta: unknown): number | undefined { + return (meta as ProviderMetadata | null | undefined)?.openrouter?.usage?.cost; +} + +export function extractCachedTokens(meta: unknown): number | undefined { + return (meta as ProviderMetadata | null | undefined)?.openrouter?.usage?.promptTokensDetails?.cachedTokens; +} + +type RawUsage = { + input_tokens?: number, + output_tokens?: number, + cache_read_input_tokens?: number, + cache_creation_input_tokens?: number, + prompt_tokens?: number, + completion_tokens?: number, + prompt_tokens_details?: { cached_tokens?: number, cache_write_tokens?: number }, + cost?: number, +}; + +type SseEvent = { + id?: string, + usage?: RawUsage, + message?: { usage?: RawUsage }, + delta?: { usage?: RawUsage }, +}; + +const emptyUsage = (): UsageFields => ({}); +const isUsageEmpty = (u: UsageFields): boolean => + u.inputTokens == null && u.outputTokens == null && u.cachedInputTokens == null + && u.cacheCreationTokens == null && u.costUsd == null + && u.generationId == null; + +function readUsageBlock(usage: RawUsage, into: UsageFields): void { + // Anthropic splits prompt tokens across three buckets; sum for parity with OpenAI's `prompt_tokens`. + if (usage.input_tokens != null) { + into.inputTokens = usage.input_tokens + (usage.cache_read_input_tokens ?? 0) + (usage.cache_creation_input_tokens ?? 0); + } else { + into.inputTokens = usage.prompt_tokens ?? into.inputTokens; + } + into.outputTokens = usage.output_tokens ?? usage.completion_tokens ?? into.outputTokens; + into.cachedInputTokens = usage.cache_read_input_tokens ?? usage.prompt_tokens_details?.cached_tokens ?? into.cachedInputTokens; + into.cacheCreationTokens = usage.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_write_tokens ?? into.cacheCreationTokens; + if (usage.cost != null) into.costUsd = usage.cost; +} + +function mergeUsageFromEvent(event: unknown, into: UsageFields): void { + if (event == null || typeof event !== "object") return; + const e = event as SseEvent; + if (e.usage) readUsageBlock(e.usage, into); + if (e.message?.usage) readUsageBlock(e.message.usage, into); + if (e.delta?.usage) readUsageBlock(e.delta.usage, into); + if (typeof e.id === "string" && e.id.length > 0 && into.generationId == null) { + into.generationId = e.id; + } +} + +export function extractOpenRouterUsage(obj: unknown): UsageFields | undefined { + const acc = emptyUsage(); + mergeUsageFromEvent(obj, acc); + return isUsageEmpty(acc) ? undefined : acc; +} + +export async function scanSseForUsage(stream: ReadableStream, signal?: AbortSignal): Promise { + const reader = stream.getReader(); + let onAbort: (() => void) | undefined; + try { + const decoder = new TextDecoder(); + const acc = emptyUsage(); + let buffer = ""; + onAbort = () => { + reader.cancel().catch(() => {}); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + const EVENT_TERMINATOR = /\r\n\r\n|\r\r|\n\n/; + const LINE_SPLITTER = /\r\n|\r|\n/; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let match: RegExpExecArray | null; + while ((match = EVENT_TERMINATOR.exec(buffer)) !== null) { + const block = buffer.slice(0, match.index); + buffer = buffer.slice(match.index + match[0].length); + for (const line of block.split(LINE_SPLITTER)) { + if (!line.startsWith("data:")) continue; + const dataStr = line.slice(5).trim(); + if (!dataStr || dataStr === "[DONE]") continue; + try { + mergeUsageFromEvent(JSON.parse(dataStr), acc); + } catch (err) { + captureError("ai-proxy-sse-parse", err); + } + } + } + } + return isUsageEmpty(acc) ? undefined : acc; + } finally { + if (onAbort) signal?.removeEventListener("abort", onAbort); + reader.releaseLock(); + } +} + +const GENERATION_RETRY_DELAYS_MS = [1500, 4000]; +const GENERATION_PER_REQUEST_TIMEOUT_MS = 5000; + +type GenerationRecord = { + id: string, + total_cost?: number, + cache_discount?: number, + upstream_inference_cost?: number, +}; + +async function fetchGenerationOnce( + generationId: string, + apiKey: string, +): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), GENERATION_PER_REQUEST_TIMEOUT_MS); + try { + const res = await fetch(`https://openrouter.ai/api/v1/generation?id=${encodeURIComponent(generationId)}`, { + headers: { "Authorization": `Bearer ${apiKey}` }, + signal: ctrl.signal, + }); + if (res.status === 404) return "not_ready"; + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`OpenRouter /generation returned ${res.status}: ${body.slice(0, 200)}`); + } + const json = await res.json() as { data?: GenerationRecord }; + return json.data ?? null; + } finally { + clearTimeout(timer); + } +} + +export async function refineGenerationCost(opts: { + generationId: string, + correlationId: string, +}): Promise { + const apiKey = getEnvVariable("STACK_OPENROUTER_API_KEY", ""); + if (!apiKey || apiKey === "FORWARD_TO_PRODUCTION") return; + const logToken = getEnvVariable("STACK_MCP_LOG_TOKEN", ""); + if (!logToken) return; + + for (let attempt = 0; attempt < GENERATION_RETRY_DELAYS_MS.length; attempt++) { + await new Promise(r => setTimeout(r, GENERATION_RETRY_DELAYS_MS[attempt])); + try { + const result = await fetchGenerationOnce(opts.generationId, apiKey); + if (result === "not_ready") continue; + if (result == null) return; + await callReducer("update_ai_query_cost", [ + logToken, + opts.correlationId, + opt(result.total_cost), + opt(result.cache_discount), + ]); + return; + } catch (err) { + if (attempt === GENERATION_RETRY_DELAYS_MS.length - 1) { + captureError("openrouter-generation-refine", err); + } + } + } +} diff --git a/apps/backend/src/lib/ai/prompts.ts b/apps/backend/src/lib/ai/prompts.ts index 53ac65bbc5..b1106359b6 100644 --- a/apps/backend/src/lib/ai/prompts.ts +++ b/apps/backend/src/lib/ai/prompts.ts @@ -519,33 +519,64 @@ follow these rules without exception: 5. Before finishing the code, mentally re-order your hooks and confirm the count and order are identical on every possible render path. -CANONICAL BAD EXAMPLE (crashes with React error #310): +CANONICAL SHAPE OF EVERY Dashboard COMPONENT: function Dashboard() { - const [users, setUsers] = React.useState(null); - if (!users) { - return ; // ← early return BEFORE the next hook - } - const [filter, setFilter] = React.useState(""); // ← this hook is skipped on first render - React.useEffect(() => { ... }, []); // ← and this one - return
...
; - } - -CANONICAL GOOD EXAMPLE: - function Dashboard() { - // All hooks first. Unconditional. Same count every render. + // 1) ALL hooks first. Unconditional. Same count every render. + // Includes React.useState / useEffect / useCallback / useMemo / useRef + // AND every DashboardUI.use* (useDataSource, etc). const [users, setUsers] = React.useState(null); const [filter, setFilter] = React.useState(""); const [error, setError] = React.useState(null); React.useEffect(() => { ... }, []); - // Conditional rendering AFTER all hooks: + // 2) Conditional rendering happens ONLY AFTER every hook has run. if (error) return ; if (!users) return ; return
...
; } -If you catch yourself writing \`if (...) return ...\` anywhere above a \`React.useXxx\` call, -STOP and move the return below every hook. +Mental check before you emit: scan your Dashboard function top-to-bottom. If ANY line +starting with \`use\` (React.useX, DashboardUI.useX, or any custom useX) sits BELOW a +\`return\`, an \`if\`, a ternary, or a loop, the code is wrong — move the hook up. + +CUSTOM HOOKS COUNT TOO — \`DashboardUI.useDataSource\` IS A HOOK +───────────────────────────────────────────────────────────────── +Anything starting with \`use\` is a hook, regardless of namespace. \`DashboardUI.useDataSource\` +and any other \`use*\` from \`DashboardUI\` follow the SAME rules as \`React.useState\` — they +MUST be called unconditionally at the top of the component, before any \`if\` / early +\`return\` / ternary / loop. The \`DashboardUI\` namespace does NOT exempt them. + +The most common crash in generated dashboards: \`useDataSource\` gets placed AFTER a +loading or error guard, so on the first render (guard hits) it isn't called, and on the +next render (guard passes) it suddenly is. Hook count changes between renders → React +error #310. NEVER put \`useDataSource\` below an early return. + + // ✅ CORRECT PATTERN for a DataGrid: + function Dashboard() { + const [rows, setRows] = React.useState(null); + const [error, setError] = React.useState(null); + const [gridState, setGridState] = React.useState(DashboardUI.createDefaultDataGridState()); + React.useEffect(() => { + stackServerApp.listUsers({ includeAnonymous: true, limit: 500 }) + .then(setRows) + .catch((e) => setError(String(e))); + }, []); + const gridData = DashboardUI.useDataSource({ // ← always called + data: rows ?? [], // ← tolerate null pre-load + columns: [/* ... */], + getRowId: (u) => u.id, + sorting: gridState.sorting, + quickSearch: gridState.quickSearch, + pagination: gridState.pagination, + paginationMode: "client", + }); + if (error) return
{error}
; + if (rows == null) return ; + return ; + } + +The rule: ALL hooks first (including \`useDataSource\`), THEN the conditional returns. +Pass \`data: rows ?? []\` so \`useDataSource\` is safe to call while \`rows\` is still null. ──────────────────────────────────────── EDITING BEHAVIOR (when existing code is provided) @@ -554,7 +585,108 @@ EDITING BEHAVIOR (when existing code is provided) - Always preserve parts of the dashboard the user didn't ask to change. - If the user asks to add something, add it without removing existing content. - If the user asks to change styling, colors, or layout, make those changes while preserving functionality. -- Always call the updateDashboard tool with the COMPLETE updated source code — no partial code or diffs. + +CHOOSING THE RIGHT TOOL — patchDashboard vs. updateDashboard +You have TWO write tools. Pick the right one. Wrong choice wastes tokens and time — +or worse, breaks the layout. + +The decision is NOT just about size. It's about whether the change is LOCAL to one +element's own attributes, or whether it ripples through surrounding layout / sibling +positioning / shared state. + +- patchDashboard — for changes that are LOCAL to one element and don't affect siblings. + Use it for: + * Rename a label or heading + * Change a color, className, or style on one element + * Swap an icon + * Tweak one prop value (e.g. limit: 100 → 500) + * Adjust one chart's config (axis label, color, format) + * Fix a single bug in one function or hook body + * Add or remove ONE self-contained leaf component (a badge, an icon) + + How it works: \`{ edits: [{ oldText, newText, occurrenceIndex? }, ...] }\` — each edit + is a literal find-and-replace on the CURRENT source. + * \`oldText\` MUST appear verbatim in the current source — copy it character-for-character + including whitespace and line breaks. No paraphrasing, no normalization. + * Include enough surrounding context in \`oldText\` to make the match UNIQUE. If the same + snippet appears more than once, either expand \`oldText\` to disambiguate OR set + \`occurrenceIndex\` (0-indexed) to pick the Nth match. + * \`newText\` is the replacement. Use \`""\` to delete. + * Batch related edits in ONE call (up to 20). Edits apply in order; later edits see the + result of earlier ones. + * If a single \`oldText\` block can hold all your changes, prefer one large edit over + many small ones — fewer matching failures. + +- updateDashboard — for changes that affect LAYOUT, ORDERING, or REGIONS, even if the + user pointed at a single component. Re-emit the full source. + Use it for: + * REORDERING components — moving a card up/down, swapping two charts, changing the + sequence of rows. The grid's child order matters; sibling indices shift. + * RESIZING — making a card wider/taller, changing grid-cols-2 to grid-cols-3, + adjusting a chart's height, changing col-span/row-span. Affects siblings' + positioning in the same grid. + * MOVING a component to a different parent or section. + * Adding a NEW row or column to a grid (sibling positions change). + * Layout overhauls (flex → grid, single column → two-column). + * Replacing the data model or switching the entire theme. + * Initial creation of a brand-new dashboard. + + The rule of thumb: ask "would this change break or shift any neighboring component?" + If yes → updateDashboard. If the change is purely cosmetic on the targeted element + itself → patchDashboard. + + Why: patchDashboard is a literal text replacement. It can't reason about JSX siblings, + grid templates, or array order. A "move this card up" patch that just swaps two JSX + blocks usually breaks because the surrounding structure (commas, fragment boundaries, + conditional renders) doesn't survive a naive swap. + + Do NOT use updateDashboard for purely cosmetic edits. Rewriting 3000 tokens to change + one className is wrong; emit a patchDashboard with one edit instead. + +WIDGET CONTEXT FROM THE USER +When the user prefixes their message with a block like: + [Widget: User Signups] + Path: div.grid > div:nth-of-type(2) > h3 + HTML:

User Signups

+they have clicked a specific element in the running dashboard. The HTML is a verbatim +slice of the rendered DOM and almost always contains text that also appears in the JSX +source — use it as your \`oldText\` anchor when patching. The Path describes the +element's position in the render tree, useful for disambiguation when the HTML alone +repeats. + +A widget pointer narrows the TARGET, but it does NOT decide the tool. Apply the same +rule above: if the user wants to restyle/rename the targeted widget → patchDashboard. +If the user wants to reorder, resize, move, or restructure the layout around the widget +→ updateDashboard. "Make this card bigger" affects the grid → updateDashboard, even +though the user only pointed at one card. + +ACTION INTENTS FROM THE USER +The user may include a block like: + [Action: Add a new component to the dashboard] +This is a structural intent — the user wants something NEW added. Adding a component +shifts sibling positions in the layout grid and is exactly the case that updateDashboard +handles (per the rule above: structural change → full rewrite). Do NOT try to express +"add a card" as a patchDashboard with a JSX fragment insertion — JSX commas, parent +containers, and grid template-cols may all need adjustment together. Re-emit the full +source via updateDashboard with the new component placed sensibly in the existing +layout, preserving everything else. + +The user's typed text describes WHAT to add ("a metric card for active users", +"a chart of weekly signups"). Combine the action intent with the typed text to decide +what to build. + +RUNTIME ERROR REPORTS FROM THE USER +The user may include a block like: + [Error: The dashboard crashed at runtime — please diagnose and fix.] + Message: + Stack: + Component stack: +The dashboard threw at runtime. Localize the bug from the stack and component stack, +identify the smallest possible fix in the source, and apply it. If the fix is a +one-line change (a typo, a wrong prop name, a missing null check), use patchDashboard. +If the fix requires restructuring (a hook ordering bug, a malformed JSX tree), use +updateDashboard. Either way, preserve the rest of the dashboard. Don't strip out +features the user didn't ask to remove just because they're near the crash site. ──────────────────────────────────────── CORE DATA FETCHING RULES (STACK) @@ -697,9 +829,16 @@ Example: return () => window.removeEventListener('chat-state-change', handler); }, []); React.useEffect(() => { - stackServerApp.listUsers({ includeAnonymous: true }) - .then(result => { setUsers(result); setLoading(false); }) - .catch(err => { setError(String(err)); setLoading(false); }); + (async () => { + try { + const result = await stackServerApp.listUsers({ includeAnonymous: true }); + setUsers(result); + setLoading(false); + } catch (err) { + setError(String(err)); + setLoading(false); + } + })(); }, []); if (loading) return
Loading...
; if (error) return
{error}
; @@ -858,11 +997,17 @@ embed the query in the dashboard itself so it fetches live data at runtime: const [rows, setRows] = React.useState(null); const [error, setError] = React.useState(null); React.useEffect(() => { - stackServerApp.queryAnalytics({ - query: "SELECT toStartOfDay(event_at) AS day, count() AS n FROM events WHERE event_at > now() - INTERVAL 30 DAY GROUP BY day ORDER BY day" - }) - .then(res => setRows(res.result)) - .catch(err => { console.error('[Dashboard] query failed', err); setError('Failed to load analytics'); }); + (async () => { + try { + const res = await stackServerApp.queryAnalytics({ + query: "SELECT toStartOfDay(event_at) AS day, count() AS n FROM events WHERE event_at > now() - INTERVAL 30 DAY GROUP BY day ORDER BY day" + }); + setRows(res.result); + } catch (err) { + console.error('[Dashboard] query failed', err); + setError('Failed to load analytics'); + } + })(); }, []); Rules: @@ -963,11 +1108,14 @@ PRE-EMIT CHECKLIST (RUN THIS IN YOUR HEAD BEFORE CALLING updateDashboard) Before you call updateDashboard, silently walk through these four checks. If any fails, fix it FIRST and re-run the list. - [1] HOOK ORDER — Are all \`React.useState\` / \`React.useEffect\` / \`React.useCallback\` calls at + [1] HOOK ORDER — Is EVERY call starting with \`use\` (React.useState / useEffect / + useCallback / useMemo / useRef AND every DashboardUI.use* including useDataSource) at the top of the Dashboard component, before every \`if\` / early \`return\` / conditional? - If no, move them up. This prevents React error #310. Also check that any variable - referenced inside a hook initializer (e.g. \`useState(() => foo(columns))\`) is declared - ABOVE that hook — a TDZ error looks like a hook-order crash but isn't one. + If no, move them up — this is the #1 cause of React error #310 in generated dashboards. + For \`useDataSource\` specifically: pass \`data: rows ?? []\` so the hook is safe to call + while data is still loading, then guard on \`rows == null\` AFTER the hook. Also check + that any variable referenced inside a hook initializer (e.g. \`useState(() => foo(columns))\`) + is declared ABOVE that hook — a TDZ error looks like a hook-order crash but isn't one. [2] DATA HONESTY — Does every field the code references actually exist in the SDK types or ClickHouse schema shown in context? No made-up field names, no hardcoded sample arrays, @@ -990,7 +1138,7 @@ FIRST and re-run the list. All five pass → emit the tool call. Any fail → fix, re-check, emit. -You MUST call the updateDashboard tool with the complete source code. NEVER output code directly in the chat. +You MUST call patchDashboard or updateDashboard per the tool-choice rules above (local edit → patchDashboard; structural/layout change → updateDashboard, with full source). NEVER output code directly in the chat. `, "run-query": ` diff --git a/apps/backend/src/lib/ai/qa-reviewer.ts b/apps/backend/src/lib/ai/qa/qa-reviewer.ts similarity index 56% rename from apps/backend/src/lib/ai/qa-reviewer.ts rename to apps/backend/src/lib/ai/qa/qa-reviewer.ts index 16d0a97de6..4167f80a1b 100644 --- a/apps/backend/src/lib/ai/qa-reviewer.ts +++ b/apps/backend/src/lib/ai/qa/qa-reviewer.ts @@ -1,9 +1,9 @@ import { createMCPClient } from "@ai-sdk/mcp"; import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; -import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { captureError, StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; import { generateText, stepCountIs } from "ai"; -import { getConnection } from "./mcp-logger"; -import { createOpenRouterProvider } from "./models"; +import { callReducer, opt } from "../spacetimedb-client"; +import { createOpenRouterProvider } from "../models"; import { getVerifiedQaContext } from "./verified-qa"; const QA_SYSTEM_PROMPT = `You are a QA reviewer for Stack Auth's AI documentation assistant. @@ -49,8 +49,14 @@ export async function reviewMcpCall(entry: { if (!apiKey || apiKey === "FORWARD_TO_PRODUCTION") { return; } + try { + await entry.logPromise; + } catch (err) { + captureError("qa-reviewer-log-wait", err); + return; + } - let devinClient: Awaited> | null = null; + let mcpClient: Awaited> | null = null; const failureUpdate = (err: unknown) => ({ qaNeedsHumanReview: true, @@ -74,21 +80,26 @@ export async function reviewMcpCall(entry: { qaErrorMessage: string | undefined, }; - try { - // Wait for the log row to be written first - await entry.logPromise; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 120_000); - devinClient = await createMCPClient({ + try { + mcpClient = await createMCPClient({ transport: { type: "http", url: "https://mcp.deepwiki.com/mcp", }, }); - const devinTools = await devinClient.tools(); + const mcpTools = await mcpClient.tools(); const openrouter = createOpenRouterProvider(); const model = openrouter(REVIEW_MODEL_ID); + const MAX_RESPONSE_CHARS = 12_000; + const truncatedResponse = entry.response.length > MAX_RESPONSE_CHARS + ? `${entry.response.slice(0, MAX_RESPONSE_CHARS)}\n\n[...truncated ${entry.response.length - MAX_RESPONSE_CHARS} chars]` + : entry.response; + const userMessage = [ "## Question", entry.question, @@ -97,7 +108,7 @@ export async function reviewMcpCall(entry: { entry.reason, "", "## AI Response", - entry.response, + truncatedResponse, ].join("\n"); const verifiedQa = await getVerifiedQaContext(); @@ -105,9 +116,11 @@ export async function reviewMcpCall(entry: { const result = await generateText({ model, system: QA_SYSTEM_PROMPT + verifiedQa, - tools: devinTools as Parameters[0]["tools"], + tools: mcpTools as Parameters[0]["tools"], stopWhen: stepCountIs(10), messages: [{ role: "user", content: userMessage }], + abortSignal: controller.signal, + maxOutputTokens: 2000, }); const conversation = result.steps.map((step, i) => { @@ -125,19 +138,19 @@ export async function reviewMcpCall(entry: { }; }); - const jsonMatch = result.text.match(/\{[\s\S]*\}/); - if (!jsonMatch) { - throw new Error("No JSON found in QA review response"); + const raw = extractJsonObject(result.text); + if (raw == null) { + throw new StackAssertionError(`No valid JSON object found in QA review response: ${result.text.slice(0, 200)}`); } - const raw = JSON.parse(jsonMatch[0]); if ( typeof raw.needsHumanReview !== "boolean" || typeof raw.answerCorrect !== "boolean" || typeof raw.answerRelevant !== "boolean" || !Array.isArray(raw.flags) || - typeof raw.overallScore !== "number" + typeof raw.overallScore !== "number" || + !Number.isFinite(raw.overallScore) ) { - throw new Error(`Invalid QA review response shape: ${JSON.stringify(raw).slice(0, 200)}`); + throw new StackAssertionError(`Invalid QA review response shape: ${JSON.stringify(raw).slice(0, 200)}`); } const parsed = raw as { needsHumanReview: boolean, @@ -148,9 +161,11 @@ export async function reviewMcpCall(entry: { overallScore: number, }; parsed.overallScore = Math.max(0, Math.min(100, Math.round(parsed.overallScore))); + const hasCriticalFlag = parsed.flags.some(f => f.severity === "critical"); + const needsHumanReview = parsed.needsHumanReview || parsed.overallScore < 50 || hasCriticalFlag; update = { - qaNeedsHumanReview: parsed.needsHumanReview, + qaNeedsHumanReview: needsHumanReview, qaAnswerCorrect: parsed.answerCorrect, qaAnswerRelevant: parsed.answerRelevant, qaFlagsJson: JSON.stringify(parsed.flags), @@ -160,25 +175,103 @@ export async function reviewMcpCall(entry: { qaErrorMessage: undefined, }; } catch (err) { - captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); + captureError("qa-reviewer-review-failed", err); update = failureUpdate(err); + } finally { + clearTimeout(timeoutId); } - if (devinClient) { - await devinClient.close().catch((err: unknown) => { - captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); - }); + if (mcpClient) { + try { + await mcpClient.close(); + } catch (err) { + captureError("qa-reviewer-mcp-client-close", err); + } } - const conn = await getConnection(); - if (!conn) return; - const token = getEnvVariable("STACK_MCP_LOG_TOKEN"); - await conn.reducers.updateMcpQaReview({ - token, - correlationId: entry.correlationId, - qaReviewModelId: REVIEW_MODEL_ID, - ...update, - }).catch((err: unknown) => { - captureError("qa-reviewer", err instanceof Error ? err : new Error(String(err))); - }); + const token = getEnvVariable("STACK_MCP_LOG_TOKEN", ""); + try { + await callReducer("update_mcp_qa_review", [ + token, + entry.correlationId, + update.qaNeedsHumanReview, + update.qaAnswerCorrect, + update.qaAnswerRelevant, + update.qaFlagsJson, + update.qaImprovementSuggestions, + update.qaOverallScore, + REVIEW_MODEL_ID, + opt(update.qaConversationJson), + opt(update.qaErrorMessage), + ]); + } catch (err) { + captureError("qa-reviewer-update-reducer-failed", err); + } +} + +function extractJsonObject(text: string): Record | null { + const tryParse = (s: string): Record | null => { + try { + const parsed = JSON.parse(s) as unknown; + return (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) + ? parsed as Record + : null; + } catch { + return null; + } + }; + + const trimmed = text.trim(); + + // 1. Whole response is JSON. + const direct = tryParse(trimmed); + if (direct) return direct; + + // 2. Inside ```json ... ``` or ``` ... ``` fence. + const fenceMatch = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/); + if (fenceMatch) { + const fenced = tryParse(fenceMatch[1].trim()); + if (fenced) return fenced; + } + + // 3. Find every balanced { ... } and try them, longest first. + const candidates: string[] = []; + for (let i = 0; i < trimmed.length; i++) { + if (trimmed[i] !== "{") continue; + let depth = 0; + let inString = false; + let escape = false; + for (let j = i; j < trimmed.length; j++) { + const c = trimmed[j]; + if (escape) { + escape = false; + continue; + } + if (c === "\\") { + escape = true; + continue; + } + if (c === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (c === "{") { + depth++; + } else if (c === "}") { + depth--; + if (depth === 0) { + candidates.push(trimmed.slice(i, j + 1)); + break; + } + } + } + } + candidates.sort((a, b) => b.length - a.length); + for (const c of candidates) { + const parsed = tryParse(c); + if (parsed) return parsed; + } + + return null; } diff --git a/apps/backend/src/lib/ai/qa/reviewer-auth.ts b/apps/backend/src/lib/ai/qa/reviewer-auth.ts new file mode 100644 index 0000000000..1f869c5da5 --- /dev/null +++ b/apps/backend/src/lib/ai/qa/reviewer-auth.ts @@ -0,0 +1,8 @@ +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +export function assertIsAiChatReviewer(user: { client_read_only_metadata?: unknown }): void { + const metadata = user.client_read_only_metadata; + if (!(metadata && typeof metadata === "object" && "isAiChatReviewer" in metadata && metadata.isAiChatReviewer === true)) { + throw new StatusError(StatusError.Forbidden, "You are not approved to perform MCP review operations."); + } +} diff --git a/apps/backend/src/lib/ai/verified-qa.ts b/apps/backend/src/lib/ai/qa/verified-qa.ts similarity index 71% rename from apps/backend/src/lib/ai/verified-qa.ts rename to apps/backend/src/lib/ai/qa/verified-qa.ts index 5f7b7a561e..a2c6b6cb03 100644 --- a/apps/backend/src/lib/ai/verified-qa.ts +++ b/apps/backend/src/lib/ai/qa/verified-qa.ts @@ -1,31 +1,31 @@ import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { Result } from "@stackframe/stack-shared/dist/utils/results"; -import { getConnection } from "./mcp-logger"; +import { callSql } from "../spacetimedb-client"; + +type VerifiedRow = { + question: string, + answer: string, +}; export async function getVerifiedQaContext(): Promise { const result = await Result.fromPromise(getVerifiedQaContextInner()); if (result.status === "error") { - captureError("verified-qa", result.error instanceof Error ? result.error : new Error(String(result.error))); + captureError("verified-qa", result.error); return ""; } return result.data; } async function getVerifiedQaContextInner(): Promise { - const conn = await getConnection(); - if (!conn) return ""; - - const pairs: Array<{ question: string; answer: string }> = []; - for (const row of conn.db.mcpCallLog.iter()) { - if (row.publishedToQa) { - pairs.push({ - question: row.humanCorrectedQuestion ?? row.question, - answer: row.humanCorrectedAnswer ?? row.response, - }); - } - } - - if (pairs.length === 0) return ""; + const rows = await callSql( + "SELECT question, answer FROM published_qa" + ); + if (rows.length === 0) return ""; + + const pairs = rows.map(row => ({ + question: row.question, + answer: row.answer, + })); const formatted = pairs.map((p, i) => `${i + 1}. Q: ${p.question}\n A: ${p.answer}` diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts deleted file mode 100644 index bf16c63e07..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/index.ts +++ /dev/null @@ -1,129 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb cli version 2.1.0 (commit 6981f48b4bc1a71c8dd9bdfe5a2c343f6370243d). - -/* eslint-disable */ -/* tslint:disable */ -import { - DbConnectionBuilder as __DbConnectionBuilder, - DbConnectionImpl as __DbConnectionImpl, - SubscriptionBuilderImpl as __SubscriptionBuilderImpl, - TypeBuilder as __TypeBuilder, - Uuid as __Uuid, - convertToAccessorMap as __convertToAccessorMap, - makeQueryBuilder as __makeQueryBuilder, - procedureSchema as __procedureSchema, - procedures as __procedures, - reducerSchema as __reducerSchema, - reducers as __reducers, - schema as __schema, - t as __t, - table as __table, - type AlgebraicTypeType as __AlgebraicTypeType, - type DbConnectionConfig as __DbConnectionConfig, - type ErrorContextInterface as __ErrorContextInterface, - type Event as __Event, - type EventContextInterface as __EventContextInterface, - type Infer as __Infer, - type QueryBuilder as __QueryBuilder, - type ReducerEventContextInterface as __ReducerEventContextInterface, - type RemoteModule as __RemoteModule, - type SubscriptionEventContextInterface as __SubscriptionEventContextInterface, - type SubscriptionHandleImpl as __SubscriptionHandleImpl, -} from "spacetimedb"; - -// Import all reducer arg schemas -import AddManualQaReducer from "./add_manual_qa_reducer"; -import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; -import LogMcpCallReducer from "./log_mcp_call_reducer"; -import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; -import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; -import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; - -// Import all procedure arg schemas - -// Import all table schema definitions -import McpCallLogRow from "./mcp_call_log_table"; - -/** Type-only namespace exports for generated type groups. */ - -/** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ -const tablesSchema = __schema({ - mcpCallLog: __table({ - name: 'mcp_call_log', - indexes: [ - { accessor: 'id', name: 'mcp_call_log_id_idx_btree', algorithm: 'btree', columns: [ - 'id', - ] }, - ], - constraints: [ - { name: 'mcp_call_log_id_key', constraint: 'unique', columns: ['id'] }, - ], - }, McpCallLogRow), -}); - -/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ -const reducersSchema = __reducers( - __reducerSchema("add_manual_qa", AddManualQaReducer), - __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), - __reducerSchema("log_mcp_call", LogMcpCallReducer), - __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), - __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), - __reducerSchema("update_mcp_qa_review", UpdateMcpQaReviewReducer), -); - -/** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ -const proceduresSchema = __procedures( -); - -/** The remote SpacetimeDB module schema, both runtime and type information. */ -const REMOTE_MODULE = { - versionInfo: { - cliVersion: "2.1.0" as const, - }, - tables: tablesSchema.schemaType.tables, - reducers: reducersSchema.reducersType.reducers, - ...proceduresSchema, -} satisfies __RemoteModule< - typeof tablesSchema.schemaType, - typeof reducersSchema.reducersType, - typeof proceduresSchema ->; - -/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */ -export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType); - -/** The reducers available in this remote SpacetimeDB module. */ -export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers); - -/** The context type returned in callbacks for all possible events. */ -export type EventContext = __EventContextInterface; -/** The context type returned in callbacks for reducer events. */ -export type ReducerEventContext = __ReducerEventContextInterface; -/** The context type returned in callbacks for subscription events. */ -export type SubscriptionEventContext = __SubscriptionEventContextInterface; -/** The context type returned in callbacks for error events. */ -export type ErrorContext = __ErrorContextInterface; -/** The subscription handle type to manage active subscriptions created from a {@link SubscriptionBuilder}. */ -export type SubscriptionHandle = __SubscriptionHandleImpl; - -/** Builder class to configure a new subscription to the remote SpacetimeDB instance. */ -export class SubscriptionBuilder extends __SubscriptionBuilderImpl {} - -/** Builder class to configure a new database connection to the remote SpacetimeDB instance. */ -export class DbConnectionBuilder extends __DbConnectionBuilder {} - -/** The typed database connection to manage connections to the remote SpacetimeDB instance. This class has type information specific to the generated module. */ -export class DbConnection extends __DbConnectionImpl { - /** Creates a new {@link DbConnectionBuilder} to configure and connect to the remote SpacetimeDB instance. */ - static builder = (): DbConnectionBuilder => { - return new DbConnectionBuilder(REMOTE_MODULE, (config: __DbConnectionConfig) => new DbConnection(config)); - }; - - /** Creates a new {@link SubscriptionBuilder} to configure a subscription to the remote SpacetimeDB instance. */ - override subscriptionBuilder = (): SubscriptionBuilder => { - return new SubscriptionBuilder(this); - }; -} - diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts deleted file mode 100644 index 772cd1fc78..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/log_mcp_call_reducer.ts +++ /dev/null @@ -1,27 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { - TypeBuilder as __TypeBuilder, - t as __t, - type AlgebraicTypeType as __AlgebraicTypeType, - type Infer as __Infer, -} from "spacetimedb"; - -export default { - token: __t.string(), - correlationId: __t.string(), - conversationId: __t.option(__t.string()), - toolName: __t.string(), - reason: __t.string(), - userPrompt: __t.string(), - question: __t.string(), - response: __t.string(), - stepCount: __t.u32(), - innerToolCallsJson: __t.string(), - durationMs: __t.u64(), - modelId: __t.string(), - errorMessage: __t.option(__t.string()), -}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts deleted file mode 100644 index bcf0dab879..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/mcp_call_log_table.ts +++ /dev/null @@ -1,44 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { - TypeBuilder as __TypeBuilder, - t as __t, - type AlgebraicTypeType as __AlgebraicTypeType, - type Infer as __Infer, -} from "spacetimedb"; - -export default __t.row({ - id: __t.u64().primaryKey(), - correlationId: __t.string().name("correlation_id"), - conversationId: __t.option(__t.string()).name("conversation_id"), - createdAt: __t.timestamp().name("created_at"), - toolName: __t.string().name("tool_name"), - reason: __t.string(), - userPrompt: __t.string().name("user_prompt"), - question: __t.string(), - response: __t.string(), - stepCount: __t.u32().name("step_count"), - innerToolCallsJson: __t.string().name("inner_tool_calls_json"), - durationMs: __t.u64().name("duration_ms"), - modelId: __t.string().name("model_id"), - errorMessage: __t.option(__t.string()).name("error_message"), - qaReviewedAt: __t.option(__t.timestamp()).name("qa_reviewed_at"), - qaNeedsHumanReview: __t.option(__t.bool()).name("qa_needs_human_review"), - qaAnswerCorrect: __t.option(__t.bool()).name("qa_answer_correct"), - qaAnswerRelevant: __t.option(__t.bool()).name("qa_answer_relevant"), - qaFlagsJson: __t.option(__t.string()).name("qa_flags_json"), - qaImprovementSuggestions: __t.option(__t.string()).name("qa_improvement_suggestions"), - qaOverallScore: __t.option(__t.u32()).name("qa_overall_score"), - qaReviewModelId: __t.option(__t.string()).name("qa_review_model_id"), - qaConversationJson: __t.option(__t.string()).name("qa_conversation_json"), - qaErrorMessage: __t.option(__t.string()).name("qa_error_message"), - humanReviewedAt: __t.option(__t.timestamp()).name("human_reviewed_at"), - humanReviewedBy: __t.option(__t.string()).name("human_reviewed_by"), - humanCorrectedQuestion: __t.option(__t.string()).name("human_corrected_question"), - humanCorrectedAnswer: __t.option(__t.string()).name("human_corrected_answer"), - publishedToQa: __t.option(__t.bool()).name("published_to_qa"), - publishedAt: __t.option(__t.timestamp()).name("published_at"), -}); diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts deleted file mode 100644 index 4af9e7b736..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/types.ts +++ /dev/null @@ -1,46 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { - TypeBuilder as __TypeBuilder, - t as __t, - type AlgebraicTypeType as __AlgebraicTypeType, - type Infer as __Infer, -} from "spacetimedb"; - -export const McpCallLog = __t.object("McpCallLog", { - id: __t.u64(), - correlationId: __t.string(), - conversationId: __t.option(__t.string()), - createdAt: __t.timestamp(), - toolName: __t.string(), - reason: __t.string(), - userPrompt: __t.string(), - question: __t.string(), - response: __t.string(), - stepCount: __t.u32(), - innerToolCallsJson: __t.string(), - durationMs: __t.u64(), - modelId: __t.string(), - errorMessage: __t.option(__t.string()), - qaReviewedAt: __t.option(__t.timestamp()), - qaNeedsHumanReview: __t.option(__t.bool()), - qaAnswerCorrect: __t.option(__t.bool()), - qaAnswerRelevant: __t.option(__t.bool()), - qaFlagsJson: __t.option(__t.string()), - qaImprovementSuggestions: __t.option(__t.string()), - qaOverallScore: __t.option(__t.u32()), - qaReviewModelId: __t.option(__t.string()), - qaConversationJson: __t.option(__t.string()), - qaErrorMessage: __t.option(__t.string()), - humanReviewedAt: __t.option(__t.timestamp()), - humanReviewedBy: __t.option(__t.string()), - humanCorrectedQuestion: __t.option(__t.string()), - humanCorrectedAnswer: __t.option(__t.string()), - publishedToQa: __t.option(__t.bool()), - publishedAt: __t.option(__t.timestamp()), -}); -export type McpCallLog = __Infer; - diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts deleted file mode 100644 index d5ac825c9a..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/types/procedures.ts +++ /dev/null @@ -1,10 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { type Infer as __Infer } from "spacetimedb"; - -// Import all procedure arg schemas - - diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts deleted file mode 100644 index 87a6606ae4..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/types/reducers.ts +++ /dev/null @@ -1,22 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { type Infer as __Infer } from "spacetimedb"; - -// Import all reducer arg schemas -import AddManualQaReducer from "../add_manual_qa_reducer"; -import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; -import LogMcpCallReducer from "../log_mcp_call_reducer"; -import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; -import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; -import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; - -export type AddManualQaParams = __Infer; -export type DeleteQaEntryParams = __Infer; -export type LogMcpCallParams = __Infer; -export type MarkHumanReviewedParams = __Infer; -export type UpdateHumanCorrectionParams = __Infer; -export type UpdateMcpQaReviewParams = __Infer; - diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts b/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts deleted file mode 100644 index 035e1d017c..0000000000 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/update_mcp_qa_review_reducer.ts +++ /dev/null @@ -1,25 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -/* eslint-disable */ -/* tslint:disable */ -import { - TypeBuilder as __TypeBuilder, - t as __t, - type AlgebraicTypeType as __AlgebraicTypeType, - type Infer as __Infer, -} from "spacetimedb"; - -export default { - token: __t.string(), - correlationId: __t.string(), - qaNeedsHumanReview: __t.bool(), - qaAnswerCorrect: __t.bool(), - qaAnswerRelevant: __t.bool(), - qaFlagsJson: __t.string(), - qaImprovementSuggestions: __t.string(), - qaOverallScore: __t.u32(), - qaReviewModelId: __t.string(), - qaConversationJson: __t.option(__t.string()), - qaErrorMessage: __t.option(__t.string()), -}; diff --git a/apps/backend/src/lib/ai/spacetimedb-client.ts b/apps/backend/src/lib/ai/spacetimedb-client.ts new file mode 100644 index 0000000000..87a1f4eda3 --- /dev/null +++ b/apps/backend/src/lib/ai/spacetimedb-client.ts @@ -0,0 +1,133 @@ +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError, StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +function httpBase(): string | null { + return getEnvVariable("STACK_SPACETIMEDB_URL", "") || null; +} + +// Cap each individual fetch to SpacetimeDB at 10s +const SPACETIMEDB_FETCH_TIMEOUT_MS = 10_000; + +let enrollmentPromise: Promise | null = null; + +async function getServiceToken(): Promise { + const base = httpBase(); + if (!base) return null; + const token = getEnvVariable("STACK_SPACETIMEDB_SERVICE_TOKEN", ""); + if (!token) return null; + const logToken = getEnvVariable("STACK_MCP_LOG_TOKEN", ""); + if (!logToken) return null; + + if (!enrollmentPromise) { + enrollmentPromise = (async () => { + try { + await rawCallReducer(token, "enroll_service", [ + logToken, + "Stack Auth Backend", + ]); + } catch (err) { + enrollmentPromise = null; + throw err; + } + })(); + } + await enrollmentPromise; + return token; +} + +async function rawCallReducer(token: string, reducer: string, args: unknown[]): Promise { + const base = httpBase(); + if (!base) throw new StackAssertionError("SpacetimeDB not configured"); + const dbName = getEnvVariable("STACK_SPACETIMEDB_DB_NAME"); + const res = await fetch(`${base}/v1/database/${encodeURIComponent(dbName)}/call/${encodeURIComponent(reducer)}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${token}`, + }, + body: JSON.stringify(args, (_, v) => { + if (typeof v !== "bigint") return v; + const MAX = BigInt(Number.MAX_SAFE_INTEGER); + if (v <= MAX && v >= -MAX) return Number(v); + return v.toString(); + }), + signal: AbortSignal.timeout(SPACETIMEDB_FETCH_TIMEOUT_MS), + }); + if (!res.ok) { + const preview = (await res.text()).slice(0, 200); + throw spacetimeDbError(`Reducer ${reducer} failed`, res.status, preview); + } +} + + +function spacetimeDbError(label: string, status: number, preview: string): Error { + const detail = `${label} (${status}): ${preview}`; + if (status >= 400 && status < 500) return new StatusError(status, detail); + if (status >= 500) return new StatusError(StatusError.BadGateway, `${label} (upstream ${status}): ${preview}`); + return new StackAssertionError(detail); +} + +export async function callReducer(reducer: string, args: unknown[]): Promise { + const token = await getServiceToken(); + if (!token) return; + await rawCallReducer(token, reducer, args); +} + +/** + * Like {@link callReducer} but throws when SpacetimeDB isn't configured, rather + * than no-opping. Use for endpoints where the client treats a 200 as proof the + * mutation actually ran (reviewer enrollment, human QA edits, deletions). + * Fire-and-forget logging paths should keep using the best-effort variant. + */ +export async function callReducerStrict(reducer: string, args: unknown[]): Promise { + const token = await getServiceToken(); + if (!token) { + throw new StackAssertionError( + `SpacetimeDB is not configured. Reducer ${reducer} cannot run. ` + + `Check STACK_SPACETIMEDB_URL and STACK_SPACETIMEDB_SERVICE_TOKEN.` + ); + } + await rawCallReducer(token, reducer, args); +} + +/** + * Wraps a nullable value in the SpacetimeDB tagged-variant encoding expected + * by HTTP reducer calls for `Option` arguments. Use for every reducer arg + * that's declared `.optional()` in the module source. + */ +export function opt(value: T | null | undefined): { some: T } | { none: [] } { + return value == null ? { none: [] } : { some: value }; +} + +export async function callSql>(sql: string): Promise { + const token = await getServiceToken(); + if (!token) return []; + const base = httpBase(); + if (!base) return []; + const dbName = getEnvVariable("STACK_SPACETIMEDB_DB_NAME"); + const res = await fetch(`${base}/v1/database/${encodeURIComponent(dbName)}/sql`, { + method: "POST", + headers: { "Authorization": `Bearer ${token}` }, + body: sql, + signal: AbortSignal.timeout(SPACETIMEDB_FETCH_TIMEOUT_MS), + }); + if (!res.ok) { + const preview = (await res.text()).slice(0, 200); + throw spacetimeDbError("SQL query failed", res.status, preview); + } + const parsed = await res.json() as Array<{ + schema: { elements: Array<{ name: { some?: string } | null }> }, + rows: unknown[][], + }>; + if (parsed.length === 0) return []; + const first = parsed[0]; + const cols = first.schema.elements.map(e => e.name?.some ?? ""); + return first.rows.map(row => { + const obj: Record = {}; + cols.forEach((c, i) => { + obj[c] = row[i]; + }); + return obj as T; + }); +} + diff --git a/apps/backend/src/lib/ai/tools/create-dashboard.ts b/apps/backend/src/lib/ai/tools/create-dashboard.ts index 380a8eb27b..2753379248 100644 --- a/apps/backend/src/lib/ai/tools/create-dashboard.ts +++ b/apps/backend/src/lib/ai/tools/create-dashboard.ts @@ -10,9 +10,28 @@ import { z } from "zod"; */ export function updateDashboardTool(auth: SmartRequestAuth | null) { return tool({ - description: "Update the dashboard with new source code. The source code must define a React functional component named 'Dashboard' (no props). It runs inside a sandboxed iframe with React, Recharts, DashboardUI, and stackServerApp available as globals. No imports, exports, or require statements.", + description: "Replace the entire dashboard source. Use ONLY for initial creation or large structural rewrites that touch most of the file. For any change smaller than ~30% of the file, use patchDashboard instead. The source must define a React functional component named 'Dashboard' (no props). Runs in a sandboxed iframe with React, Recharts, DashboardUI, and stackServerApp as globals. No imports, exports, or require statements.", inputSchema: z.object({ content: z.string().describe("The complete updated JSX source code for the Dashboard component"), }), }); } + +/** + * Tool for surgical edits to the existing dashboard source. + * + * Like updateDashboardTool, this is inert server-side - the call streams back to the + * client, which applies the patches against currentTsxSource and updates state. + */ +export function patchDashboardTool(auth: SmartRequestAuth | null) { + return tool({ + description: "Apply one or more surgical text edits to the existing dashboard source. Prefer this over updateDashboard for any change smaller than ~30% of the file (rename, restyle, add/remove a single component, fix one bug). Each edit is a literal find-and-replace on the current source. Returns nothing - the client applies the patch.", + inputSchema: z.object({ + edits: z.array(z.object({ + oldText: z.string().min(1).describe("Exact substring to find in the current source. Must match verbatim including whitespace. Include enough surrounding context to make the match unique, OR set occurrenceIndex when oldText repeats."), + newText: z.string().describe("Replacement text. Empty string deletes the match."), + occurrenceIndex: z.number().int().min(0).optional().describe("0-indexed match to replace when oldText appears multiple times. Omit when oldText is unique in the source."), + })).min(1).max(20).describe("Edits applied in order against the running source. Later edits see the result of earlier ones."), + }), + }); +} diff --git a/apps/backend/src/lib/ai/tools/index.ts b/apps/backend/src/lib/ai/tools/index.ts index bb571e2cab..ba0d522264 100644 --- a/apps/backend/src/lib/ai/tools/index.ts +++ b/apps/backend/src/lib/ai/tools/index.ts @@ -1,6 +1,7 @@ import { SmartRequestAuth } from "@/route-handlers/smart-request"; +import { StackAssertionError, captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { ToolSet } from "ai"; -import { updateDashboardTool } from "./create-dashboard"; +import { patchDashboardTool, updateDashboardTool } from "./create-dashboard"; import { createEmailDraftTool } from "./create-email-draft"; import { createEmailTemplateTool } from "./create-email-template"; import { createEmailThemeTool } from "./create-email-theme"; @@ -14,8 +15,10 @@ export const TOOL_NAMES = [ "create-email-template", "create-email-draft", "update-dashboard", + "patch-dashboard" ] as const; -export type ToolName = typeof TOOL_NAMES[number]; +export type ToolName = typeof TOOL_NAMES[number] + export type ToolContext = { auth: SmartRequestAuth | null, @@ -37,10 +40,7 @@ export async function getTools( } case "sql-query": { - const sqlTool = createSqlQueryTool(context.auth, context.targetProjectId); - if (sqlTool != null) { - tools["queryAnalytics"] = sqlTool; - } + tools["queryAnalytics"] = createSqlQueryTool(context.auth, context.targetProjectId); break; } @@ -64,13 +64,39 @@ export async function getTools( break; } + case "patch-dashboard": { + tools["patchDashboard"] = patchDashboardTool(context.auth); + break; + } + default: { - // TypeScript will ensure this is unreachable if we handle all cases const _exhaustive: never = toolName; - console.warn(`Unknown tool name: ${_exhaustive}`); + captureError("ai-tools-getTools", new StackAssertionError(`Unknown tool name: ${_exhaustive as string}`)); } } } return tools; } + +/** + * Validates that all requested tool names are valid. + * Throws an error if any tool name is invalid. + */ +export function validateToolNames(toolNames: unknown): toolNames is ToolName[] { + if (!Array.isArray(toolNames)) { + return false; + } + + const validToolNames: ToolName[] = [ + "docs", + "sql-query", + "create-email-theme", + "create-email-template", + "create-email-draft", + "update-dashboard", + "patch-dashboard" + ]; + + return toolNames.every((name) => validToolNames.includes(name as ToolName)); +} diff --git a/apps/backend/src/lib/ai/tools/sql-query.ts b/apps/backend/src/lib/ai/tools/sql-query.ts index 37054d02b9..5821f643d7 100644 --- a/apps/backend/src/lib/ai/tools/sql-query.ts +++ b/apps/backend/src/lib/ai/tools/sql-query.ts @@ -9,7 +9,18 @@ export const SQL_QUERY_RESULT_MAX_CHARS = 50_000; export function createSqlQueryTool(auth: SmartRequestAuth | null, targetProjectId?: string | null) { if (auth == null) { - return null; + // Return a stub tool that surfaces the auth requirement to the model as a tool + // result, instead of throwing. This way the model can react gracefully (e.g. tell + // the user to sign in) rather than the request failing with a 4xx the model never sees. + return tool({ + description: "Run analytics SQL queries. Currently unavailable: this tool requires the user to be signed in. If the user asks an analytics question, explain that they need to sign in first instead of calling this tool.", + inputSchema: z.object({ + query: z.string(), + }), + execute: async () => ({ + error: "Authentication required. The user is not signed in, so analytics queries cannot run. Inform the user that they need to sign in to access analytics.", + }), + }); } const projectId = targetProjectId ?? auth.tenancy.project.id; diff --git a/apps/backend/src/lib/ai/types.ts b/apps/backend/src/lib/ai/types.ts new file mode 100644 index 0000000000..fe511c9e09 --- /dev/null +++ b/apps/backend/src/lib/ai/types.ts @@ -0,0 +1,47 @@ +import type { selectModel } from "@/lib/ai/models"; +import type { Json } from "@stackframe/stack-shared/dist/utils/json"; +import type { ModelMessage, ToolSet } from "ai"; + +export type ContentBlock = + | { type: "text", text: string } + | { + type: "tool-call", + toolName: string, + toolCallId: string, + args: Json, + argsText: string, + result: Json, + }; + +export type McpCallMetadata = { + toolName: string, + reason: string, + userPrompt: string, + conversationId?: string | null, +}; + +export type MessageLike = { role: string, content: unknown }; + +export type CommonLogFields = { + correlationId: string, + mode: "stream" | "generate", + systemPromptId: string, + quality: string, + speed: string, + modelId: string, + isAuthenticated: boolean, + projectId: string | undefined, + userId: string | undefined, + requestedToolsJson: string, + messagesJson: string, + conversationId: string | undefined, +}; + +export type ModeContext = { + model: ReturnType, + cachedMessages: ModelMessage[], + toolsArg: ToolSet | undefined, + stepLimit: number, + common: CommonLogFields, + startedAt: number, +}; diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/dashboards/[dashboardId]/page-client.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/dashboards/[dashboardId]/page-client.tsx index 17ad8f25bb..4bb17d7162 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/dashboards/[dashboardId]/page-client.tsx +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/dashboards/[dashboardId]/page-client.tsx @@ -13,23 +13,27 @@ import { DashboardToolUI, type AssistantComposerApi, } from "@/components/vibe-coding"; -import { ToolCallContent } from "@/components/vibe-coding/chat-adapters"; +import { ToolCallContent, type DashboardChip, type DashboardPatchFailure, type DashboardPatchSnapshot } from "@/components/vibe-coding/chat-adapters"; +import { patchSnapshotKey, registerPatchSnapshot } from "@/components/vibe-coding/dashboard-tool-components"; +import type { AppId } from "@/lib/apps-frontend"; import { useUpdateConfig } from "@/lib/config-update"; +import { getPublicEnvVar } from "@/lib/env"; import { cn } from "@/lib/utils"; import { ChatCircleIcon, + CursorClickIcon, FloppyDiskIcon, PencilSimpleIcon, TrashIcon, + WarningIcon, XIcon, } from "@phosphor-icons/react"; +import { useUser } from "@stackframe/stack"; import { ALL_APPS } from "@stackframe/stack-shared/dist/apps/apps-config"; -import { typedEntries } from "@stackframe/stack-shared/dist/utils/objects"; import { throwErr } from "@stackframe/stack-shared/dist/utils/errors"; -import type { AppId } from "@/lib/apps-frontend"; +import { typedEntries } from "@stackframe/stack-shared/dist/utils/objects"; import { runAsynchronouslyWithAlert } from "@stackframe/stack-shared/dist/utils/promises"; -import { getPublicEnvVar } from "@/lib/env"; -import { useUser } from "@stackframe/stack"; +import { generateUuid } from "@stackframe/stack-shared/dist/utils/uuids"; import { usePathname } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { PageLayout } from "../../page-layout"; @@ -144,8 +148,9 @@ function DashboardDetailContent({ // Coalesce duplicate error reports — React re-renders a crashed component several times, // and uncaught-error listeners can fire twice for the same exception. We only surface the - // first unique error per 2-second window so the composer isn't stomped on repeatedly. + // first unique error per 2-second window so the chip bar isn't spammed. const lastErrorRef = useRef<{ signature: string, at: number } | null>(null); + const handleDashboardRuntimeError = useCallback( (err: DashboardRuntimeError) => { const signature = `${err.message}::${(err.stack ?? "").slice(0, 200)}`; @@ -155,55 +160,74 @@ function DashboardDetailContent({ } lastErrorRef.current = { signature, at: now }; - // Build a compact fix-request prompt. We keep the stack to ~1200 chars so the - // agent gets enough context to localize the bug without drowning in frame noise. - const stackSlice = (err.stack ?? "").trim().slice(0, 1200); - const componentStackSlice = (err.componentStack ?? "").trim().slice(0, 400); - const prefill = [ - "The dashboard just crashed at runtime. Please diagnose and fix it.", - "", - "Error:", - err.message, - stackSlice ? `\nStack:\n${stackSlice}` : "", - componentStackSlice ? `\nComponent stack:${componentStackSlice}` : "", - ] - .filter(Boolean) - .join("\n"); - - // Open the chat panel if it's closed so the user sees the pre-filled composer. - // The iframe panel doesn't unmount when chat toggles, so no reload cost. setIsChatOpen(true); - composerApiRef.current?.setText(prefill); + + const errorChip: DashboardChip = { + kind: "error", + id: generateUuid(), + message: err.message, + stack: err.stack, + componentStack: err.componentStack, + }; + setPendingChips((prev) => [...prev, errorChip]); + + const api = composerApiRef.current; + if (api && api.getText().trim().length === 0) { + api.setText("could you please fix this error"); + } toast({ variant: "destructive", title: "Dashboard crashed", - description: "Error added to chat — hit send to fix it.", + description: "Error added as a chip — hit send to fix it.", }); }, [toast], ); + const [pendingChips, setPendingChips] = useState([]); + const pendingChipsRef = useRef([]); + useEffect(() => { + pendingChipsRef.current = pendingChips; + }, [pendingChips]); + + const getPendingChips = useCallback(() => pendingChipsRef.current, []); + const consumePendingChips = useCallback(() => { + pendingChipsRef.current = []; + setPendingChips([]); + }, []); + const removePendingChip = useCallback((id: string) => { + setPendingChips((prev) => { + const next = prev.filter((c) => c.id !== id); + pendingChipsRef.current = next; + return next; + }); + }, []); + const handleWidgetSelected = useCallback( (selection: WidgetSelection) => { - const api = composerApiRef.current; - if (!api) return; - setIsChatOpen(true); - - const { heading, tagName, rect, textPreview } = selection.metadata; - const name = heading ?? `${tagName} (${rect.width}×${rect.height})`; - const domContext = [ - `[Widget: ${name}]`, - textPreview ? `Content: ${textPreview.slice(0, 200)}` : "", - ].filter(Boolean).join("\n"); - - const currentText = api.getText(); - api.setText(domContext + "\n" + currentText); + const { heading, selectorPath, outerHTMLSnippet } = selection.metadata; + const name = (heading && heading.trim().length > 0 && heading.trim().length <= 60) + ? heading.trim() + : "Widget"; + + setPendingChips((prev) => [ + ...prev, + { kind: "widget", id: generateUuid(), name, selectorPath, outerHTMLSnippet }, + ]); }, [], ); + const handleAddComponent = useCallback(() => { + setIsChatOpen(true); + setPendingChips((prev) => { + if (prev.some((c) => c.kind === "action-add-component")) return prev; + return [...prev, { kind: "action-add-component", id: generateUuid() }]; + }); + }, []); + useEffect(() => { if (!hasUnsavedChanges) return; setNeedConfirm(true); @@ -252,6 +276,33 @@ function DashboardDetailContent({ } }, []); + const handlePatchApplied = useCallback((updatedSource: string, failures: DashboardPatchFailure[], snapshots: DashboardPatchSnapshot[]) => { + setPendingCode(updatedSource); + setCurrentTsxSource(updatedSource); + clearTimeout(codePhaseTimerRef.current); + setCodePhase("typing"); + codePhaseTimerRef.current = setTimeout(() => { + setCodePhase("loading"); + codePhaseTimerRef.current = setTimeout(() => { + setCodePhase("done"); + }, 1000); + }, 3000); + for (const snap of snapshots) { + registerPatchSnapshot(patchSnapshotKey(snap.edits), snap.resultSource); + } + if (failures.length > 0) { + const summary = failures.slice(0, 3).map((f) => + `#${f.index + 1} ${f.reason} ("${f.oldTextPreview}${f.oldTextPreview.length >= 80 ? "…" : ""}")`, + ).join("; "); + const remainder = failures.length > 3 ? ` (+${failures.length - 3} more)` : ""; + toast({ + variant: "destructive", + title: `${failures.length} ${failures.length === 1 ? "edit" : "edits"} didn't apply`, + description: `${summary}${remainder}. Ask the AI to retry with more context.`, + }); + } + }, [toast]); + const handleRunStart = useCallback(() => { setIsGenerating(true); setPendingCode(null); @@ -366,6 +417,7 @@ function DashboardDetailContent({ onReady={handleIframeReady} onRuntimeError={handleDashboardRuntimeError} onWidgetSelected={handleWidgetSelected} + onAddComponentClicked={handleAddComponent} isChatOpen={isChatOpen} /> @@ -391,7 +443,7 @@ function DashboardDetailContent({ {/* Both panels are always in the DOM so the iframe never unmounts/reloads. The chat panel animates its width; the dashboard panel adjusts via flex-1. */} -
+
{/* Dashboard iframe panel */}
0 + ? + : undefined + } historyAdapter={createHistoryAdapter(adminApp, dashboardId)} toolComponents={} useOffWhiteLightMode @@ -494,6 +551,64 @@ const DASHBOARD_COMPOSER_PLACEHOLDER = { ], } as const; +function ChipBar({ + chips, + onRemove, +}: { + chips: DashboardChip[], + onRemove: (id: string) => void, +}) { + return ( +
+ {chips.map((c) => { + if (c.kind === "widget") { + return ( + + ); + } + if (c.kind === "action-add-component") { + return ( + + ); + } + // error + return ( + + ); + })} +
+ ); +} + function ChatPanelHeader({ displayName, isEditingName, diff --git a/apps/dashboard/src/components/assistant-ui/thread.tsx b/apps/dashboard/src/components/assistant-ui/thread.tsx index af27c79e4b..8de0f7204e 100644 --- a/apps/dashboard/src/components/assistant-ui/thread.tsx +++ b/apps/dashboard/src/components/assistant-ui/thread.tsx @@ -62,7 +62,8 @@ export const Thread: FC<{ runningStatusMessages?: string[], composerAttachments?: boolean, attachmentAdapter?: AttachmentAdapter, -}> = ({ useOffWhiteLightMode = false, composerPlaceholder, hideMessageActions = false, runningStatusMessages, composerAttachments = false, attachmentAdapter }) => { + composerTopContent?: React.ReactNode, +}> = ({ useOffWhiteLightMode = false, composerPlaceholder, hideMessageActions = false, runningStatusMessages, composerAttachments = false, attachmentAdapter, composerTopContent }) => { return ( @@ -110,7 +111,7 @@ export const Thread: FC<{ : "from-background via-background", )}> - +
@@ -537,11 +538,12 @@ const ComposerStaticInput: FC<{ placeholder?: string }> = ({ placeholder }) => { ); }; -const Composer: FC<{ placeholder?: ComposerPlaceholder }> = ({ placeholder }) => { +const Composer: FC<{ placeholder?: ComposerPlaceholder, topContent?: React.ReactNode }> = ({ placeholder, topContent }) => { const attachmentsEnabled = useComposerAttachmentsEnabled(); return ( {attachmentsEnabled && } + {topContent} {typeof placeholder === "object" ? ( @@ -284,6 +333,13 @@ function getSandboxDocument(artifact: DashboardArtifact, baseUrl: string, dashbo // Controls visibility flag — only true in the full dashboard viewer (not cmd+K preview) window.__showControls = ${showControls}; window.__chatOpen = ${initialChatOpen}; + // Inline + + + `; } @@ -605,6 +754,12 @@ export type WidgetSelection = { classes: string, textPreview: string, rect: { width: number, height: number }, + /** CSS-style selector chain from #root down to the clicked widget. Capped at 10 + segments. Lets the AI ground a patch on a real chunk of structure. */ + selectorPath: string, + /** First ~300 chars of the widget's outerHTML — verbatim rendered markup the AI + can match against when locating the JSX node in source. */ + outerHTMLSnippet: string, }, }; @@ -616,6 +771,7 @@ export const DashboardSandboxHost = memo(function DashboardSandboxHost({ onReady, onRuntimeError, onWidgetSelected, + onAddComponentClicked, isChatOpen, }: { artifact: DashboardArtifact, @@ -628,6 +784,9 @@ export const DashboardSandboxHost = memo(function DashboardSandboxHost({ onRuntimeError?: (err: DashboardRuntimeError) => void, /** Fires when the user clicks "Add to chat" on a widget overlay in the iframe. */ onWidgetSelected?: (selection: WidgetSelection) => void, + /** Fires when the user clicks the in-iframe "Add a component" button at the bottom + of the dashboard. Parent pushes an action chip into the composer chip bar. */ + onAddComponentClicked?: () => void, isChatOpen?: boolean, }) { const iframeRef = useRef(null); @@ -643,6 +802,8 @@ export const DashboardSandboxHost = memo(function DashboardSandboxHost({ onRuntimeErrorRef.current = onRuntimeError; const onWidgetSelectedRef = useRef(onWidgetSelected); onWidgetSelectedRef.current = onWidgetSelected; + const onAddComponentClickedRef = useRef(onAddComponentClicked); + onAddComponentClickedRef.current = onAddComponentClicked; const user = useUser({ or: "redirect" }); const { resolvedTheme } = useTheme(); @@ -768,11 +929,18 @@ export const DashboardSandboxHost = memo(function DashboardSandboxHost({ width: typeof event.data.metadata?.rect?.width === "number" ? event.data.metadata.rect.width : 0, height: typeof event.data.metadata?.rect?.height === "number" ? event.data.metadata.rect.height : 0, }, + selectorPath: typeof event.data.metadata?.selectorPath === "string" ? event.data.metadata.selectorPath : "", + outerHTMLSnippet: typeof event.data.metadata?.outerHTMLSnippet === "string" ? event.data.metadata.outerHTMLSnippet : "", }, }); return; } + if (type === "dashboard-add-component-clicked") { + onAddComponentClickedRef.current?.(); + return; + } + if (type === "stack-ai-dashboard-ready") { onReadyRef.current?.(); return; diff --git a/apps/dashboard/src/components/vibe-coding/assistant-chat.tsx b/apps/dashboard/src/components/vibe-coding/assistant-chat.tsx index aa9ad1e339..6b222f2e9d 100644 --- a/apps/dashboard/src/components/vibe-coding/assistant-chat.tsx +++ b/apps/dashboard/src/components/vibe-coding/assistant-chat.tsx @@ -32,6 +32,8 @@ type AssistantChatProps = { runningStatusMessages?: string[], /** Enable image attachment UI (subject to shared MAX_IMAGES_PER_MESSAGE/MAX_IMAGE_BYTES_PER_FILE). */ composerAttachments?: boolean, + /** Content rendered inside the composer box, above the textarea. Used for widget chips. */ + composerTopContent?: React.ReactNode, /** * Called once the composer runtime is mounted. Parent stores the handle in a ref * and can then call `setText(...)` imperatively. Fires inside the runtime provider @@ -64,6 +66,7 @@ export default function AssistantChat({ hideMessageActions = false, runningStatusMessages, composerAttachments = false, + composerTopContent, onComposerReady, }: AssistantChatProps) { const attachmentAdapter = useMemo( @@ -92,6 +95,7 @@ export default function AssistantChat({ runningStatusMessages={runningStatusMessages} composerAttachments={composerAttachments} attachmentAdapter={attachmentAdapter} + composerTopContent={composerTopContent} /> {toolComponents} diff --git a/apps/dashboard/src/components/vibe-coding/chat-adapters.ts b/apps/dashboard/src/components/vibe-coding/chat-adapters.ts index d05baf31e7..4746637d8a 100644 --- a/apps/dashboard/src/components/vibe-coding/chat-adapters.ts +++ b/apps/dashboard/src/components/vibe-coding/chat-adapters.ts @@ -25,6 +25,123 @@ const isToolCall = (content: { type: string }): content is ToolCallContent => { return content.type === "tool-call"; }; +export type DashboardWidgetContext = { + kind: "widget", + id: string, + name: string, + selectorPath: string, + outerHTMLSnippet: string, +}; + +export type DashboardActionContext = { + kind: "action-add-component", + id: string, +}; + +export type DashboardErrorContext = { + kind: "error", + id: string, + message: string, + stack?: string, + componentStack?: string, +}; + +export type DashboardChip = + | DashboardWidgetContext + | DashboardActionContext + | DashboardErrorContext; + +export type DashboardPatchEdit = { + oldText: string, + newText: string, + occurrenceIndex?: number, +}; + +export type DashboardPatchFailure = { + index: number, + reason: "not-found" | "ambiguous", + oldTextPreview: string, +}; + +export type DashboardPatchResult = { + updatedSource: string, + applied: number, + failures: DashboardPatchFailure[], +}; + +export type DashboardPatchSnapshot = { + edits: DashboardPatchEdit[], + resultSource: string, +}; + +export function applyDashboardPatches(source: string, edits: DashboardPatchEdit[]): DashboardPatchResult { + let draft = source; + let applied = 0; + const failures: DashboardPatchFailure[] = []; + + edits.forEach((edit, index) => { + const preview = edit.oldText.slice(0, 80).replace(/\s+/g, " "); + + const matches: number[] = []; + let from = 0; + while (from <= draft.length) { + const at = draft.indexOf(edit.oldText, from); + if (at === -1) break; + matches.push(at); + from = at + Math.max(edit.oldText.length, 1); + } + + if (matches.length === 0) { + failures.push({ index, reason: "not-found", oldTextPreview: preview }); + return; + } + + let chosenIndex: number; + if (edit.occurrenceIndex != null) { + if ( + !Number.isInteger(edit.occurrenceIndex) + || edit.occurrenceIndex < 0 + || edit.occurrenceIndex >= matches.length + ) { + failures.push({ index, reason: "not-found", oldTextPreview: preview }); + return; + } + chosenIndex = matches[edit.occurrenceIndex]; + } else if (matches.length > 1) { + failures.push({ index, reason: "ambiguous", oldTextPreview: preview }); + return; + } else { + chosenIndex = matches[0]; + } + + draft = draft.slice(0, chosenIndex) + edit.newText + draft.slice(chosenIndex + edit.oldText.length); + applied += 1; + }); + + if (failures.length > 0) { + return { updatedSource: source, applied: 0, failures }; + } + return { updatedSource: draft, applied, failures }; +} + +function parsePatchEdits(args: unknown): DashboardPatchEdit[] | null { + if (typeof args !== "object" || args === null) return null; + const editsRaw = (args as { edits?: unknown }).edits; + if (!Array.isArray(editsRaw)) return null; + const edits: DashboardPatchEdit[] = []; + for (const e of editsRaw) { + if (typeof e !== "object" || e === null) return null; + const { oldText, newText, occurrenceIndex } = e as { oldText?: unknown, newText?: unknown, occurrenceIndex?: unknown }; + if (typeof oldText !== "string" || typeof newText !== "string") return null; + edits.push({ + oldText, + newText, + occurrenceIndex: typeof occurrenceIndex === "number" ? occurrenceIndex : undefined, + }); + } + return edits; +} + /** Maps thread messages to the backend wire format; merges `attachments` into `content`. */ function formatThreadMessagesForBackend( messages: readonly { role: string, content: readonly { type: string }[], attachments?: readonly { content?: readonly unknown[] }[] }[], @@ -250,8 +367,8 @@ export async function* streamDashboardCode( // Only give the agent the sql-query tool when we know which project to scope it to. // Without projectId, the tool would fall back to the internal project — wrong target. const tools = options?.projectId - ? ["update-dashboard", "sql-query"] - : ["update-dashboard"]; + ? ["update-dashboard", "patch-dashboard", "sql-query"] + : ["update-dashboard", "patch-dashboard"]; const chunkStream = await sendAiStreamRequest( backendBaseUrl, @@ -296,8 +413,8 @@ export async function generateDashboardCode( options?.enabledAppIds, ); const tools = options?.projectId - ? ["update-dashboard", "sql-query"] - : ["update-dashboard"]; + ? ["update-dashboard", "patch-dashboard", "sql-query"] + : ["update-dashboard", "patch-dashboard"]; const rawContent = await sendAiRequest( backendBaseUrl, currentUser, @@ -391,14 +508,46 @@ export function createDashboardChatAdapter( projectId?: string, onRunStart?: () => void, onRunEnd?: () => void, + onPatchApplied?: (updatedSource: string, failures: DashboardPatchFailure[], snapshots: DashboardPatchSnapshot[]) => void, + getPendingChips?: () => DashboardChip[], + consumePendingChips?: () => void, ): ChatModelAdapter { return { async *run({ messages, abortSignal }: ChatModelRunOptions): AsyncGenerator { onRunStart?.(); try { const formattedMessages = formatThreadMessagesForBackend(messages); + const chips = getPendingChips?.() ?? []; + if (chips.length > 0) { + const chipBlock = chips.map((c) => { + if (c.kind === "widget") { + return `[Widget: ${c.name}]\nPath: ${c.selectorPath}\nHTML: ${c.outerHTMLSnippet}`; + } + if (c.kind === "action-add-component") { + return `[Action: Add a new component to the dashboard]`; + } + // Bound stack/componentStack so a 5KB trace can't blow up the prompt. + const stackSlice = c.stack ? `\nStack:\n${c.stack.slice(0, 1200)}` : ""; + const componentStackSlice = c.componentStack + ? `\nComponent stack:${c.componentStack.slice(0, 400)}` + : ""; + return `[Error: The dashboard crashed at runtime — please diagnose and fix.]\nMessage: ${c.message}${stackSlice}${componentStackSlice}`; + }).join("\n\n"); + + for (let i = formattedMessages.length - 1; i >= 0; i--) { + if (formattedMessages[i].role !== "user") continue; + const orig = formattedMessages[i].content; + const chipPart = { type: "text" as const, text: chipBlock }; + formattedMessages[i] = { + ...formattedMessages[i], + content: Array.isArray(orig) ? [chipPart, ...orig] : [chipPart], + }; + break; + } + } let latestContent: ChatContent = []; + let chipsConsumed = chips.length === 0; for await (const content of streamDashboardCode( backendBaseUrl, currentUser, @@ -410,15 +559,45 @@ export function createDashboardChatAdapter( projectId, }, )) { + if (!chipsConsumed) { + consumePendingChips?.(); + chipsConsumed = true; + } latestContent = content; yield { content }; } - const finalToolCall = latestContent.find( - (item): item is ToolCallContent => isToolCall(item) && item.toolName === "updateDashboard", - ); - if (finalToolCall) { - onToolCall(finalToolCall); + let runningSource = currentTsxSource; + let lastFullReplacement: ToolCallContent | null = null; + const aggregatedFailures: DashboardPatchFailure[] = []; + const snapshots: DashboardPatchSnapshot[] = []; + let anyPatchApplied = false; + + for (const item of latestContent) { + if (!isToolCall(item)) continue; + if (item.toolName === "updateDashboard") { + if (typeof item.args?.content === "string") { + runningSource = item.args.content; + lastFullReplacement = item; + } + } else if (item.toolName === "patchDashboard") { + const edits = parsePatchEdits(item.args); + if (!edits) continue; + const result = applyDashboardPatches(runningSource, edits); + runningSource = result.updatedSource; + anyPatchApplied = true; + snapshots.push({ edits, resultSource: runningSource }); + for (const f of result.failures) { + aggregatedFailures.push(f); + } + } + } + + if (lastFullReplacement) { + onToolCall(lastFullReplacement); + } + if (anyPatchApplied || aggregatedFailures.length > 0) { + onPatchApplied?.(runningSource, aggregatedFailures, snapshots); } } catch (error) { if (abortSignal.aborted) { diff --git a/apps/dashboard/src/components/vibe-coding/dashboard-tool-components.tsx b/apps/dashboard/src/components/vibe-coding/dashboard-tool-components.tsx index 488359cf22..01a05f1d27 100644 --- a/apps/dashboard/src/components/vibe-coding/dashboard-tool-components.tsx +++ b/apps/dashboard/src/components/vibe-coding/dashboard-tool-components.tsx @@ -17,6 +17,7 @@ import { CheckCircleIcon, EyeIcon, MagnifyingGlassIcon, + PencilSimpleIcon, WarningIcon, } from "@phosphor-icons/react"; import { useEffect, useMemo, useState, useSyncExternalStore } from "react"; @@ -36,6 +37,47 @@ function useCurrentCode() { return useSyncExternalStore(subscribe, () => currentCodeRef); } +/* ──────────────────────────────────────────────────────────────────────────── + * Patch snapshot registry. Each successful patchDashboard call stores the + * post-patch source under a stable key (the JSON-serialized edits array). The + * patch tool's chat row reads this to offer a "restore this version" action. + * + * Limitation: snapshots live in-memory and only cover patches applied during + * the current session. Patches loaded from saved chat history won't have a + * snapshot until the user re-runs them — the dialog still shows the edit text + * either way. Cross-reload restore would require persisting the result source + * with the tool call, which the inert-tool architecture doesn't currently do. + * ────────────────────────────────────────────────────────────────────────── */ +const patchSnapshots = new Map(); +const patchSnapshotListeners: Set<() => void> = new Set(); + +export function registerPatchSnapshot(editsKey: string, resultSource: string) { + patchSnapshots.set(editsKey, resultSource); + for (const l of patchSnapshotListeners) l(); +} + +function subscribePatchSnapshots(listener: () => void) { + patchSnapshotListeners.add(listener); + return () => { + patchSnapshotListeners.delete(listener); + }; +} + +function usePatchSnapshot(editsKey: string): string | undefined { + return useSyncExternalStore( + subscribePatchSnapshots, + () => patchSnapshots.get(editsKey), + ); +} + +export function patchSnapshotKey(edits: unknown): string { + try { + return JSON.stringify(edits); + } catch { + return ""; + } +} + function ToolRender({ args, isRunning }: { args: { content: string }, isRunning: boolean }) { const currentCode = useCurrentCode(); const isActive = args.content === currentCode; @@ -114,6 +156,121 @@ const ToolUI = makeAssistantToolUI< render: (props) => , }); +type PatchEditArg = { oldText?: string, newText?: string, occurrenceIndex?: number }; +type PatchToolArgs = { edits?: PatchEditArg[] }; + +function PatchToolRender({ args, isRunning }: { args: PatchToolArgs, isRunning: boolean }) { + const [open, setOpen] = useState(false); + const edits = Array.isArray(args.edits) ? args.edits : []; + const count = edits.length; + const currentCode = useCurrentCode(); + const snapshot = usePatchSnapshot(patchSnapshotKey(edits)); + // Snapshot may be missing for patches loaded from saved chat history (the in-memory + // map only covers the current session). When absent, hide the restore button rather + // than offering something we can't fulfill. + const canRestore = snapshot !== undefined && !isRunning; + const isActive = snapshot !== undefined && snapshot === currentCode; + const label = isRunning + ? "Editing dashboard..." + : count === 0 + ? "Edited dashboard" + : `Edited dashboard · ${count} ${count === 1 ? "change" : "changes"}`; + + return ( + <> +
+ + {canRestore && !isActive && ( + + + + )} +
+ + + + + Dashboard edits + + {isRunning ? "Streaming edits from the model…" : `${count} ${count === 1 ? "edit" : "edits"} applied to the source.`} + + + +
+ {edits.length === 0 && ( +
Waiting for edits…
+ )} + {edits.map((edit, i) => ( +
+
+ Edit {i + 1}{typeof edit.occurrenceIndex === "number" ? ` · occurrence ${edit.occurrenceIndex}` : ""} +
+
+
+                      {edit.oldText ?? ""}
+                    
+
+                      {edit.newText ?? ""}
+                    
+
+
+ ))} +
+
+
+
+ + ); +} + +const PatchToolUI = makeAssistantToolUI< + PatchToolArgs, + "success" +>({ + toolName: "patchDashboard", + render: (props) => , +}); + /* ──────────────────────────────────────────────────────────────────────────── * queryAnalytics tool UI — inspection steps the agent takes before/between * writes. Visual weight is DELIBERATELY lighter than the updateDashboard card: @@ -363,6 +520,7 @@ export const DashboardToolUI = ({ setCurrentCode, currentCode }: DashboardToolUI return ( <> + ); diff --git a/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts b/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts index d008d628d4..bca8de2303 100644 --- a/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts +++ b/apps/dashboard/src/lib/ai-dashboard/shared-prompt.ts @@ -1,6 +1,16 @@ import { BUNDLED_DASHBOARD_UI_TYPES, BUNDLED_TYPE_DEFINITIONS } from "@/generated/bundled-type-definitions"; import { ALL_APPS_FRONTEND, type AppId, getItemPath, hasNavigationItems } from "@/lib/apps-frontend"; -import { buildStackAuthHeaders, type CurrentUser } from "@/lib/api-headers"; +import type { CurrentUser } from "@/lib/api-headers"; + +export type DashboardMessagePart = { + type: "text", + text: string, + providerOptions?: Record, +}; +export type DashboardMessage = { + role: string, + content: string | DashboardMessagePart[], +}; /** * Builds a formatted list of available dashboard routes based on enabled apps. @@ -17,7 +27,7 @@ export function buildAvailableRoutes(enabledAppIds: AppId[]): string { routes.push({ path: "/project-settings", label: "Project Settings" }); // Dynamic routes from enabled apps - for (const appId of enabledAppIds) { + for (const appId of [...enabledAppIds].sort()) { const appFrontend = ALL_APPS_FRONTEND[appId as keyof typeof ALL_APPS_FRONTEND]; if (!hasNavigationItems(appFrontend)) { continue; @@ -35,74 +45,8 @@ export function buildAvailableRoutes(enabledAppIds: AppId[]): string { return `\nAVAILABLE DASHBOARD ROUTES (use ONLY these with window.dashboardNavigate):\n${routeList}\nDo NOT use any paths not listed above.`; } -export async function selectRelevantFiles( - prompt: string, - backendBaseUrl: string, - currentUser?: CurrentUser, -): Promise { - const availableFiles = BUNDLED_TYPE_DEFINITIONS.map((f: { path: string }) => f.path); - - const systemPromptText = `You are a code assistant helping to generate dashboard code for Stack Auth. - -Your task is to select which Stack SDK type definition files you'll need to generate the requested dashboard. - -IMPORTANT GUIDELINES: -- DO NOT be conservative in file selection - when in doubt, INCLUDE the file -- If a file might be relevant to the dashboard, SELECT IT -- For user/team dashboards: select users and/or teams files -- For project info: select projects files -- Always select server-app.ts as it contains the main SDK interface -- It's better to include extra files than to miss necessary types - -Available files: -${availableFiles.map(f => `- ${f}`).join('\n')} - -Respond with ONLY a JSON object: { "selectedFiles": ["file1.ts", "file2.ts"] } -No markdown, no explanation — just the JSON.`; - - try { - const authHeaders = await buildStackAuthHeaders(currentUser); - const response = await fetch(`${backendBaseUrl}/api/latest/ai/query/generate`, { - method: "POST", - headers: { "content-type": "application/json", ...authHeaders }, - body: JSON.stringify({ - quality: "dumb", - speed: "fast", - systemPrompt: "command-center-ask-ai", - tools: [], - messages: [{ - role: "user", - content: `${systemPromptText}\n\nDashboard request: "${prompt}"\n\nWhich type definition files do you need? When uncertain, err on the side of INCLUDING more files rather than fewer.`, - }], - }), - }); - - const result = await response.json() as { content?: Array<{ type: string, text?: string }> }; - const content = Array.isArray(result.content) ? result.content : []; - const textBlock = content.find((b) => b.type === "text"); - const responseText = textBlock?.text; - - if (!responseText) { - return availableFiles; - } - - const jsonMatch = responseText.match(/\{[\s\S]*"selectedFiles"[\s\S]*\}/); - if (!jsonMatch) { - return availableFiles; - } - - const parsed = JSON.parse(jsonMatch[0]) as { selectedFiles?: string[] }; - if (!Array.isArray(parsed.selectedFiles) || parsed.selectedFiles.length === 0) { - return availableFiles; - } - - const selected = parsed.selectedFiles.filter((f) => availableFiles.includes(f)); - - return selected; - } catch (e) { - console.log("[selectRelevantFiles] failed, returning all files:", e); - return availableFiles; - } +function getAllTypeDefinitionFiles(): string[] { + return BUNDLED_TYPE_DEFINITIONS.map((f: { path: string }) => f.path); } function stripComments(source: string): string { @@ -134,33 +78,16 @@ ${fileContents.join('\n')} `.trim(); } -function extractUserPromptText(messages: Array<{ role: string, content: unknown }>): string { - const lastUserMessage = [...messages].reverse().find(m => m.role === "user"); - if (typeof lastUserMessage?.content === "string") { - return lastUserMessage.content; - } - if (Array.isArray(lastUserMessage?.content)) { - const textPart = (lastUserMessage.content as Array<{ type: string, text?: string }>).find(c => c.type === "text"); - return textPart?.text ?? "dashboard"; - } - return "dashboard"; -} - -export async function buildDashboardMessages( - backendBaseUrl: string, - currentUser: CurrentUser | undefined, - messages: Array<{ role: string, content: unknown }>, +export function buildDashboardMessages( + _backendBaseUrl: string, + _currentUser: CurrentUser | undefined, + _messages: Array<{ role: string, content: unknown }>, currentSource?: string, enabledAppIds?: AppId[], -): Promise> { - const promptForFileSelection = extractUserPromptText(messages); - const selectedFiles = await selectRelevantFiles(promptForFileSelection, backendBaseUrl, currentUser); - const typeDefinitions = loadSelectedTypeDefinitions(selectedFiles); - +): Promise { + const typeDefinitions = loadSelectedTypeDefinitions(getAllTypeDefinitionFiles()); const availableRoutes = enabledAppIds ? buildAvailableRoutes(enabledAppIds) : ""; - const contextMessages: Array<{ role: string, content: string }> = []; - const dashboardUiDocsHeader = [ "DashboardUI component documentation (READ THIS BEFORE USING ANY COMPONENT):", "", @@ -172,27 +99,45 @@ export async function buildDashboardMessages( "bare type signatures are NOT enough to use the components correctly.", ].join("\n"); + const cachedText = `Here are the type definitions for the Stack SDK:\n${typeDefinitions}\n\n${dashboardUiDocsHeader}\n${BUNDLED_DASHBOARD_UI_TYPES}`; + const contextMessages: DashboardMessage[] = []; + + contextMessages.push({ + role: "user", + content: [ + { + type: "text", + text: cachedText, + providerOptions: { + openrouter: { cacheControl: { type: "ephemeral" } }, + }, + }, + ], + }); + contextMessages.push({ + role: "assistant", + content: "I have the SDK reference material and UI component types. What dashboard would you like me to create or edit?", + }); + + const tailParts: string[] = []; + if (availableRoutes) { + tailParts.push(availableRoutes.trimStart()); + } if (currentSource != null && currentSource.length > 0) { + tailParts.push(`Here is the current dashboard source code:\n\`\`\`tsx\n${currentSource}\n\`\`\``); + } + if (tailParts.length > 0) { contextMessages.push({ role: "user", - content: `Here is the current dashboard source code:\n\`\`\`tsx\n${currentSource}\n\`\`\`\n\nHere are the type definitions:\n${typeDefinitions}\n\n${dashboardUiDocsHeader}\n${BUNDLED_DASHBOARD_UI_TYPES}${availableRoutes}`, - }); - contextMessages.push({ - role: "assistant", - content: "I understand the current dashboard code, type definitions, and available routes. I've also read the JSDoc on every DashboardUI component I plan to use and will follow each component's canonical pattern. What changes would you like to make?", - }); - } else { - contextMessages.push({ - role: "user", - content: `Here are the type definitions for the Stack SDK:\n${typeDefinitions}\n\n${dashboardUiDocsHeader}\n${BUNDLED_DASHBOARD_UI_TYPES}${availableRoutes}`, + content: tailParts.join("\n\n"), }); contextMessages.push({ role: "assistant", - content: "I have the type definitions and available routes. I've also read the JSDoc on every DashboardUI component I plan to use and will follow each component's canonical pattern. What dashboard would you like me to create?", + content: "Got it. What changes would you like me to make?", }); } - return contextMessages; + return Promise.resolve(contextMessages); } export { BUNDLED_DASHBOARD_UI_TYPES }; diff --git a/apps/e2e/tests/backend/backend-helpers.ts b/apps/e2e/tests/backend/backend-helpers.ts index 7365aefd22..6468984a5b 100644 --- a/apps/e2e/tests/backend/backend-helpers.ts +++ b/apps/e2e/tests/backend/backend-helpers.ts @@ -1570,6 +1570,30 @@ export namespace User { } return users; } + + export async function setClientReadOnlyMetadata(userId: string, metadata: Record) { + const response = await niceBackendFetch(`/api/v1/users/${userId}`, { + method: "PATCH", + accessType: "server", + body: { + client_read_only_metadata: metadata, + }, + }); + expect(response).toMatchObject({ status: 200 }); + return response; + } +} + +export namespace AiChatReviewer { + export async function createReviewer() { + const { userId, accessToken, refreshToken } = await Auth.fastSignUp(); + await User.setClientReadOnlyMetadata(userId, { isAiChatReviewer: true }); + return { userId, accessToken, refreshToken }; + } + + export async function createNonReviewer() { + return await Auth.fastSignUp(); + } } diff --git a/apps/e2e/tests/backend/endpoints/api/latest/internal/mcp-review.test.ts b/apps/e2e/tests/backend/endpoints/api/latest/internal/mcp-review.test.ts new file mode 100644 index 0000000000..b723999bfa --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/latest/internal/mcp-review.test.ts @@ -0,0 +1,77 @@ +import { it } from "../../../../../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../../../../backend-helpers"; + +// Every mcp-review endpoint shares the same auth gate (isAiChatReviewer metadata check) +// and the same short-circuit order: auth → metadata → yup → reducer. Tests here cover +// the first three. +const endpoints = [ + { + path: "/api/latest/internal/mcp-review/mark-reviewed", + validBody: { correlationId: "abc123" }, + invalidBody: {}, + }, + { + path: "/api/latest/internal/mcp-review/unmark-reviewed", + validBody: { correlationId: "abc123" }, + invalidBody: {}, + }, + { + path: "/api/latest/internal/mcp-review/update-correction", + validBody: { + correlationId: "abc123", + correctedQuestion: "q", + correctedAnswer: "a", + publish: false, + }, + invalidBody: { correlationId: "abc123", publish: "yes" as unknown as boolean }, + }, + { + path: "/api/latest/internal/mcp-review/add-manual", + validBody: { question: "q", answer: "a", publish: false }, + invalidBody: { question: "q" }, + }, + { + path: "/api/latest/internal/mcp-review/update-qa-entry", + validBody: { qaId: "1", question: "q", answer: "a", publish: false }, + invalidBody: { qaId: "1", publish: "yes" as unknown as boolean }, + }, + { + path: "/api/latest/internal/mcp-review/delete", + validBody: { qaId: "1" }, + invalidBody: {}, + }, +] as const; + +for (const { path, validBody, invalidBody } of endpoints) { + it(`${path}: rejects unauthenticated requests`, async ({ expect }) => { + const response = await niceBackendFetch(path, { + method: "POST", + accessType: "client", + body: validBody, + }); + // yup schema on createSmartRouteHandler requires auth.user; missing auth fails + // shape validation as 400 before reaching the handler's auth-specific error path. + expect([400, 401]).toContain(response.status); + }); + + it(`${path}: rejects a signed-in user without isAiChatReviewer metadata`, async ({ expect }) => { + await AiChatReviewer.createNonReviewer(); + const response = await niceBackendFetch(path, { + method: "POST", + accessType: "client", + body: validBody, + }); + expect(response.status).toBe(403); + expect(String(response.body)).toContain("not approved to perform MCP review operations"); + }); + + it(`${path}: rejects a reviewer sending an invalid body`, async ({ expect }) => { + await AiChatReviewer.createReviewer(); + const response = await niceBackendFetch(path, { + method: "POST", + accessType: "client", + body: invalidBody, + }); + expect(response.status).toBe(400); + }); +} diff --git a/apps/e2e/tests/backend/endpoints/api/latest/internal/spacetimedb-enroll-reviewer.test.ts b/apps/e2e/tests/backend/endpoints/api/latest/internal/spacetimedb-enroll-reviewer.test.ts new file mode 100644 index 0000000000..053acf407e --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/latest/internal/spacetimedb-enroll-reviewer.test.ts @@ -0,0 +1,58 @@ +import { it } from "../../../../../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../../../../backend-helpers"; + +const VALID_HEX = "a".repeat(64); + +it("rejects unauthenticated requests", async ({ expect }) => { + const response = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: VALID_HEX }, + }); + // createSmartRouteHandler's yup schema requires auth.user; missing auth trips + // schema validation before the handler runs, so this comes back as 400, not 401. + expect([400, 401]).toContain(response.status); +}); + +it("rejects a signed-in user without isAiChatReviewer metadata", async ({ expect }) => { + await AiChatReviewer.createNonReviewer(); + const response = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: VALID_HEX }, + }); + expect(response.status).toBe(403); + expect(String(response.body)).toContain("not approved to perform MCP review operations"); +}); + +it("rejects a reviewer sending a non-hex identity", async ({ expect }) => { + await AiChatReviewer.createReviewer(); + const response = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: "not-a-hex-identity" }, + }); + expect(response.status).toBe(400); + expect(String(response.body)).toContain("Invalid identity"); +}); + +it("rejects a reviewer sending a hex identity of the wrong length", async ({ expect }) => { + await AiChatReviewer.createReviewer(); + const response = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: "a".repeat(63) }, + }); + expect(response.status).toBe(400); + expect(String(response.body)).toContain("Invalid identity"); +}); + +it("rejects a reviewer sending a request without an identity field", async ({ expect }) => { + await AiChatReviewer.createReviewer(); + const response = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: {}, + }); + expect(response.status).toBe(400); +}); diff --git a/apps/e2e/tests/spacetimedb/helpers.ts b/apps/e2e/tests/spacetimedb/helpers.ts new file mode 100644 index 0000000000..73225adae1 --- /dev/null +++ b/apps/e2e/tests/spacetimedb/helpers.ts @@ -0,0 +1,270 @@ +// All traffic uses the +// SpacetimeDB HTTP API (POST /v1/identity, /v1/database/{db}/call/{reducer}, +// /v1/database/{db}/sql) — avoids pulling the `spacetimedb` client SDK into +// the e2e package just for a handful of subscriptions. + +export type MintedIdentity = { + token: string, + identity: string, +}; + +type SqlRow = Record; + +export type SpacetimedbConfig = { + baseUrl: string, + dbName: string, + logToken: string | null, +}; + +export function getSpacetimedbConfig(): SpacetimedbConfig { + return { + baseUrl: process.env.STACK_SPACETIMEDB_URL ?? "", + dbName: process.env.STACK_SPACETIMEDB_DB_NAME ?? "stack-auth-llm", + logToken: process.env.STACK_MCP_LOG_TOKEN ?? null, + }; +} + +export async function isSpacetimedbReachable(): Promise { + const { baseUrl } = getSpacetimedbConfig(); + if (!baseUrl) return false; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 2000); + try { + const res = await fetch(`${baseUrl}/v1/identity`, { + method: "POST", + signal: controller.signal, + }); + return res.ok; + } catch (err) { + const isAbort = err instanceof DOMException && err.name === "AbortError"; + const isNetwork = err instanceof TypeError; + if (isAbort || isNetwork) return false; + throw err; + } finally { + clearTimeout(timeout); + } +} + +export async function mintIdentity(): Promise { + const { baseUrl } = getSpacetimedbConfig(); + const res = await fetch(`${baseUrl}/v1/identity`, { method: "POST" }); + if (!res.ok) throw new Error(`mintIdentity failed: HTTP ${res.status}`); + const body = await res.json() as { token: string, identity: string }; + const identity = body.identity.startsWith("0x") ? body.identity.slice(2) : body.identity; + return { token: body.token, identity }; +} + +/** + * SpacetimeDB encodes `.optional()` fields as a tagged sum type — clients must + * send `{ some: value }` or `{ none: [] }`, not raw null. Mirrors the `opt()` + * helper in apps/backend/src/lib/ai/mcp-logger.ts:87. + */ +export function opt(value: T | null | undefined): { some: T } | { none: [] } { + return value == null ? { none: [] } : { some: value }; +} + +export type ReducerCallResult = { + status: number, + ok: boolean, + body: string, +}; + +export async function callReducer( + token: string, + reducer: string, + args: unknown[], +): Promise { + const { baseUrl, dbName } = getSpacetimedbConfig(); + const res = await fetch(`${baseUrl}/v1/database/${encodeURIComponent(dbName)}/call/${encodeURIComponent(reducer)}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${token}`, + }, + body: JSON.stringify(args, (_, v) => { + if (typeof v !== "bigint") return v; + const MAX = BigInt(Number.MAX_SAFE_INTEGER); + if (v <= MAX && v >= -MAX) return Number(v); + return v.toString(); + }), + }); + return { status: res.status, ok: res.ok, body: await res.text() }; +} + +export type SqlQueryResult = { + columns: string[], + rows: SqlRow[], +}; + +/** + * Look up the `correlationId` of a freshly-inserted row by its unique `question`. + * Caller must pass a SpacetimeDB token for an enrolled operator — only they can + * read `my_visible_mcp_call_log`. Returns undefined if no match is found. + */ +export async function findCorrelationIdByQuestion( + token: string, + question: string, +): Promise { + const { rows } = await sqlQuery(token, "SELECT * FROM my_visible_mcp_call_log"); + const match = rows.find(r => r.question === question); + if (!match) return undefined; + const raw = match.correlation_id ?? match.correlationId; + return typeof raw === "string" ? raw : undefined; +} + +/** + * Look up the qa_entries primary key (qaId, a u64 bigint) for a row whose + * `sourceMcpCorrelationId` matches the given correlationId. Returns undefined + * if no qa_entries row exists yet (the call hasn't been curated). + */ +export async function findQaEntryIdBySource( + token: string, + correlationId: string, +): Promise { + const { rows } = await sqlQuery(token, "SELECT * FROM my_visible_qa_entries"); + const match = rows.find(r => { + const src = r.source_mcp_correlation_id ?? r.sourceMcpCorrelationId; + if (src == null || typeof src !== "object") return false; + const opt = src as { some?: string } | { none?: unknown }; + return "some" in opt && opt.some === correlationId; + }); + if (!match) return undefined; + return coerceBigInt(match.id); +} + +/** + * Look up a manually-added qa_entries row (sourceMcpCorrelationId is none) by + * its question text. Used by test cleanup to find rows inserted by add_manual_qa. + */ +export async function findManualQaEntryIdByQuestion( + token: string, + question: string, +): Promise { + const { rows } = await sqlQuery(token, "SELECT * FROM my_visible_qa_entries"); + const match = rows.find(r => { + if (r.question !== question) return false; + const src = r.source_mcp_correlation_id ?? r.sourceMcpCorrelationId; + if (src == null) return true; + if (typeof src === "object" && "none" in src) return true; + return false; + }); + if (!match) return undefined; + return coerceBigInt(match.id); +} + +function coerceBigInt(raw: unknown): bigint | undefined { + if (typeof raw === "string") return BigInt(raw); + if (typeof raw === "number") return BigInt(raw); + if (typeof raw === "bigint") return raw; + return undefined; +} + +/** + * Per-test collector for anything these tests drop into SpacetimeDB so + * `afterEach` can wipe it. Without this, each CI run would accumulate + * stale operators, mcp_call_log rows, and ai_query_log rows against the + * shared scratch DB. + * + * Deletions use the log token (skip if unavailable). MCP call log rows are + * looked up by their `question` marker — callers should pass a unique marker + * (`Date.now() + random` is enough). Cleanup is best-effort: individual + * failures are swallowed so one bad row doesn't leave the rest behind. + */ +export type CleanupScope = { + trackIdentity: (identity: string) => void, + trackMcpQuestion: (question: string) => void, + trackAiQueryCorrelationId: (correlationId: string) => void, + cleanup: () => Promise, +}; + +export function createCleanupScope(): CleanupScope { + const identities = new Set(); + const questions = new Set(); + const aiQueryCorrelationIds = new Set(); + + return { + trackIdentity: (identity) => { identities.add(identity); }, + trackMcpQuestion: (question) => { questions.add(question); }, + trackAiQueryCorrelationId: (correlationId) => { aiQueryCorrelationIds.add(correlationId); }, + async cleanup() { + const { logToken } = getSpacetimedbConfig(); + if (!logToken) { + identities.clear(); + questions.clear(); + aiQueryCorrelationIds.clear(); + return; + } + + const caller = await mintIdentity().catch(() => null); + if (caller == null) return; + + try { + await callReducer(caller.token, "add_operator", [ + logToken, + [`0x${caller.identity}`], + `__cleanup__-${caller.identity}`, + "Cleanup Scope", + ]).catch(() => undefined); + + for (const question of questions) { + const cid = await findCorrelationIdByQuestion(caller.token, question).catch(() => undefined); + if (cid) { + const qaId = await findQaEntryIdBySource(caller.token, cid).catch(() => undefined); + if (qaId != null) { + await callReducer(caller.token, "delete_qa_entry", [logToken, qaId]).catch(() => undefined); + } + await callReducer(caller.token, "delete_mcp_call_log", [logToken, cid]).catch(() => undefined); + } + const manualQaId = await findManualQaEntryIdByQuestion(caller.token, question).catch(() => undefined); + if (manualQaId != null) { + await callReducer(caller.token, "delete_qa_entry", [logToken, manualQaId]).catch(() => undefined); + } + } + + for (const correlationId of aiQueryCorrelationIds) { + await callReducer(caller.token, "delete_ai_query_log", [logToken, correlationId]).catch(() => undefined); + } + + for (const identity of identities) { + await callReducer(caller.token, "remove_operator", [logToken, [`0x${identity}`]]).catch(() => undefined); + } + } finally { + await callReducer(caller.token, "remove_operator", [logToken, [`0x${caller.identity}`]]).catch(() => undefined); + identities.clear(); + questions.clear(); + aiQueryCorrelationIds.clear(); + } + }, + }; +} + +export async function sqlQuery(token: string, sql: string): Promise { + const { baseUrl, dbName } = getSpacetimedbConfig(); + const res = await fetch(`${baseUrl}/v1/database/${encodeURIComponent(dbName)}/sql`, { + method: "POST", + headers: { + "Content-Type": "text/plain", + "Authorization": `Bearer ${token}`, + }, + body: sql, + }); + if (!res.ok) { + throw new Error(`SQL ${JSON.stringify(sql)} failed: HTTP ${res.status} ${await res.text()}`); + } + const payload = await res.json() as Array<{ + schema: { elements: Array<{ name: { some: string } | { none: null } }> }, + rows: unknown[][], + }>; + // `/sql` returns an array of query results (one per statement). We only send one. + if (payload.length === 0) return { columns: [], rows: [] }; + const first = payload[0]; + const columns = first.schema.elements.map(el => "some" in el.name ? el.name.some : ""); + const rows: SqlRow[] = first.rows.map(tuple => { + const obj: SqlRow = {}; + columns.forEach((c, i) => { + obj[c] = tuple[i]; + }); + return obj; + }); + return { columns, rows }; +} diff --git a/apps/e2e/tests/spacetimedb/operators-rls.test.ts b/apps/e2e/tests/spacetimedb/operators-rls.test.ts new file mode 100644 index 0000000000..0f13695537 --- /dev/null +++ b/apps/e2e/tests/spacetimedb/operators-rls.test.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { callReducer, createCleanupScope, getSpacetimedbConfig, isSpacetimedbReachable, mintIdentity, sqlQuery, type CleanupScope } from "./helpers"; + +const canRun = await isSpacetimedbReachable(); +const { logToken } = getSpacetimedbConfig(); + +describe.skipIf(!canRun)("operators table RLS", () => { + let scope: CleanupScope; + beforeEach(() => { + scope = createCleanupScope(); + }); + afterEach(async () => { + await scope.cleanup(); + }); + + it("each reviewer sees only their own operators row", async ({ expect }) => { + const a = await mintIdentity(); + scope.trackIdentity(a.identity); + await AiChatReviewer.createReviewer(); + const enrollA = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: a.identity }, + }); + expect(enrollA.status).toBe(200); + + const b = await mintIdentity(); + scope.trackIdentity(b.identity); + await AiChatReviewer.createReviewer(); + const enrollB = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: b.identity }, + }); + expect(enrollB.status).toBe(200); + + const asA = await sqlQuery(a.token, "SELECT * FROM operators"); + const asB = await sqlQuery(b.token, "SELECT * FROM operators"); + + expect(asA.rows.length).toBe(1); + expect(asB.rows.length).toBe(1); + // Different reviewers must see different (own) rows — if RLS broke, both would see two. + expect(JSON.stringify(asA.rows[0])).not.toEqual(JSON.stringify(asB.rows[0])); + }); + + it("a freshly-minted non-operator identity sees zero operators rows", async ({ expect }) => { + // Seed at least one operator so the table isn't empty. + const seeded = await mintIdentity(); + scope.trackIdentity(seeded.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: seeded.identity }, + }); + expect(enroll.status).toBe(200); + + const stranger = await mintIdentity(); + const { rows } = await sqlQuery(stranger.token, "SELECT * FROM operators"); + expect(rows.length).toBe(0); + }); + + it("enrolling a second identity as the same reviewer sweeps the first", async ({ expect }) => { + // The add_operator reducer's sweep logic deletes prior rows with the same + // stackUserId before inserting a new identity — a reviewer switching browsers + // should not accumulate stale operator rows. + const x = await mintIdentity(); + scope.trackIdentity(x.identity); + await AiChatReviewer.createReviewer(); + const enrollX = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: x.identity }, + }); + expect(enrollX.status).toBe(200); + + // Same reviewer (backendContext.userAuth unchanged) enrolls a second identity. + const y = await mintIdentity(); + scope.trackIdentity(y.identity); + const enrollY = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: y.identity }, + }); + expect(enrollY.status).toBe(200); + + // X should no longer be in operators — sweep removed its row. + const asX = await sqlQuery(x.token, "SELECT * FROM operators"); + expect(asX.rows.length).toBe(0); + // Y should still be the active operator. + const asY = await sqlQuery(y.token, "SELECT * FROM operators"); + expect(asY.rows.length).toBe(1); + }); + + it.skipIf(!logToken)( + "remove_operator reducer revokes an operator's view access", + async ({ expect }) => { + const target = await mintIdentity(); + scope.trackIdentity(target.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: target.identity }, + }); + expect(enroll.status).toBe(200); + + // Confirm enrolled. + const before = await sqlQuery(target.token, "SELECT * FROM operators"); + expect(before.rows.length).toBe(1); + + // Directly call remove_operator with the log token. + const caller = await mintIdentity(); + const removed = await callReducer(caller.token, "remove_operator", [ + logToken!, + [`0x${target.identity}`], + ]); + expect(removed.ok).toBe(true); + + // Target is no longer an operator. + const after = await sqlQuery(target.token, "SELECT * FROM operators"); + expect(after.rows.length).toBe(0); + }, + ); +}); diff --git a/apps/e2e/tests/spacetimedb/private-tables.test.ts b/apps/e2e/tests/spacetimedb/private-tables.test.ts new file mode 100644 index 0000000000..2ef64f8d17 --- /dev/null +++ b/apps/e2e/tests/spacetimedb/private-tables.test.ts @@ -0,0 +1,150 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { callReducer, createCleanupScope, getSpacetimedbConfig, isSpacetimedbReachable, mintIdentity, opt, sqlQuery, type CleanupScope } from "./helpers"; + +const canRun = await isSpacetimedbReachable(); +const { logToken } = getSpacetimedbConfig(); + +function uniqueMarker(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +describe.skipIf(!canRun)("private log tables and view gating", () => { + let scope: CleanupScope; + beforeEach(() => { + scope = createCleanupScope(); + }); + afterEach(async () => { + await scope.cleanup(); + }); + + // my_visible_ai_query_log is the counterpart to my_visible_mcp_call_log. Seeding + // requires the log token (no user-facing endpoint writes to ai_query_log), so + // skip when unavailable rather than asserting against an empty table. + it.skipIf(!logToken)( + "a freshly-minted non-operator identity sees zero rows in my_visible_ai_query_log", + async ({ expect }) => { + const seeder = await mintIdentity(); + const aiQueryCorrelationId = uniqueMarker("ai-query-corr"); + scope.trackAiQueryCorrelationId(aiQueryCorrelationId); + const seed = await callReducer(seeder.token, "log_ai_query", [ + logToken!, + aiQueryCorrelationId, + "chat", + "system-prompt-id", + "high", + "fast", + "some-model", + false, + opt(null), + opt(null), + "[]", + "[]", + "[]", + "final text", + opt(null), + opt(null), + opt(null), + opt(null), + 0, + 0n, + opt(null), + opt(null), + opt(null), + ]); + expect(seed.ok).toBe(true); + + const stranger = await mintIdentity(); + const { rows } = await sqlQuery(stranger.token, "SELECT * FROM my_visible_ai_query_log"); + expect(rows.length).toBe(0); + }, + ); + + it("cannot subscribe to the private mcp_call_log table directly", async ({ expect }) => { + // Seed a row so the table isn't empty — we're testing access control, not emptiness. + const seeder = await mintIdentity(); + scope.trackIdentity(seeder.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: seeder.identity }, + }); + expect(enroll.status).toBe(200); + const seedMarker = uniqueMarker("private-mcp-seed"); + scope.trackMcpQuestion(seedMarker); + const seed = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: seedMarker, answer: "a", publish: false }, + }); + expect(seed.status).toBe(200); + + // Private table: SpacetimeDB should either reject the query outright or return + // zero rows to non-operators. Either outcome is acceptable — the invariant is + // "the caller does not see any private-table rows." If rejection, the error + // must come from our own sqlQuery helper's HTTP-4xx path against this exact + // table (not a network blip, not a helper regression). + const stranger = await mintIdentity(); + const result = await sqlQuery(stranger.token, "SELECT * FROM mcp_call_log") + .then(r => ({ ok: true as const, rows: r.rows })) + .catch(err => ({ ok: false as const, err })); + if (result.ok) { + expect(result.rows.length).toBe(0); + } else { + expect(result.err).toBeInstanceOf(Error); + expect((result.err as Error).message).toMatch( + /SQL\s+"SELECT \* FROM mcp_call_log"\s+failed: HTTP 4\d\d/, + ); + } + }); + + it.skipIf(!logToken)( + "cannot subscribe to the private ai_query_log table directly", + async ({ expect }) => { + const seeder = await mintIdentity(); + const aiQueryCorrelationId = uniqueMarker("ai-query-corr"); + scope.trackAiQueryCorrelationId(aiQueryCorrelationId); + const seed = await callReducer(seeder.token, "log_ai_query", [ + logToken!, + aiQueryCorrelationId, + "chat", + "system-prompt-id", + "high", + "fast", + "some-model", + false, + opt(null), + opt(null), + "[]", + "[]", + "[]", + "final text", + opt(null), + opt(null), + opt(null), + opt(null), + 0, + 0n, + opt(null), + opt(null), + opt(null), + ]); + expect(seed.ok).toBe(true); + + const stranger = await mintIdentity(); + const result = await sqlQuery(stranger.token, "SELECT * FROM ai_query_log") + .then(r => ({ ok: true as const, rows: r.rows })) + .catch(err => ({ ok: false as const, err })); + if (result.ok) { + expect(result.rows.length).toBe(0); + } else { + expect(result.err).toBeInstanceOf(Error); + expect((result.err as Error).message).toMatch( + /SQL\s+"SELECT \* FROM ai_query_log"\s+failed: HTTP 4\d\d/, + ); + } + }, + ); +}); diff --git a/apps/e2e/tests/spacetimedb/published-qa-projection.test.ts b/apps/e2e/tests/spacetimedb/published-qa-projection.test.ts new file mode 100644 index 0000000000..3358ac792f --- /dev/null +++ b/apps/e2e/tests/spacetimedb/published-qa-projection.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { createCleanupScope, isSpacetimedbReachable, mintIdentity, sqlQuery, type CleanupScope } from "./helpers"; + +const canRun = await isSpacetimedbReachable(); + +const EXPECTED_PUBLISHED_QA_COLUMNS = ["id", "question", "answer", "published_at"] as const; + +// Fields from mcp_call_log that MUST NOT appear in the public view. If any of these +// reappear, the projection has regressed and internal metadata is leaking to the +// unauthenticated /questions page. +const FORBIDDEN_COLUMNS = [ + "human_reviewed_by", + "human_reviewed_at", + "user_prompt", + "qa_reviewed_at", + "qa_flags_json", + "qa_improvement_suggestions", + "qa_conversation_json", + "model_id", + "correlation_id", + "conversation_id", + "response", + "reason", + "tool_name", + "inner_tool_calls_json", + "human_corrected_question", + "human_corrected_answer", +]; + +describe.skipIf(!canRun)("published_qa view projection", () => { + let scope: CleanupScope; + beforeEach(() => { + scope = createCleanupScope(); + }); + afterEach(async () => { + await scope.cleanup(); + }); + + it("exposes only {id, question, answer, publishedAt} — no reviewer or QA internals", async ({ expect }) => { + const reviewerIdentity = await mintIdentity(); + scope.trackIdentity(reviewerIdentity.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewerIdentity.identity }, + }); + expect(enroll.status).toBe(200); + + const markerQuestion = `test-projection-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + scope.trackMcpQuestion(markerQuestion); + const markerAnswer = "answer-for-projection-test"; + + const publish = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: markerQuestion, answer: markerAnswer, publish: true }, + }); + expect(publish.status).toBe(200); + + // Query with a fresh non-operator token — published_qa is anonymousView so any + // bearer works; using a stranger identity matches the real public-page scenario. + const stranger = await mintIdentity(); + const result = await sqlQuery(stranger.token, "SELECT * FROM published_qa"); + + // Regression: the projected column set must be exactly these four names. + expect([...result.columns].sort()).toEqual([...EXPECTED_PUBLISHED_QA_COLUMNS].sort()); + for (const forbidden of FORBIDDEN_COLUMNS) { + expect(result.columns).not.toContain(forbidden); + } + + // Our marker row must be present and carry the corrected (answer) payload, not + // the raw response (which would be empty for a manually-added row). + const ours = result.rows.find(r => r.question === markerQuestion); + expect(ours).toBeDefined(); + expect(ours).toMatchObject({ question: markerQuestion, answer: markerAnswer }); + }); +}); diff --git a/apps/e2e/tests/spacetimedb/published-qa-visibility.test.ts b/apps/e2e/tests/spacetimedb/published-qa-visibility.test.ts new file mode 100644 index 0000000000..efccfca627 --- /dev/null +++ b/apps/e2e/tests/spacetimedb/published-qa-visibility.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { createCleanupScope, findManualQaEntryIdByQuestion, isSpacetimedbReachable, mintIdentity, sqlQuery, type CleanupScope } from "./helpers"; + +const canRun = await isSpacetimedbReachable(); + +function uniqueMarker(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +async function publishedQaContains(question: string): Promise { + const stranger = await mintIdentity(); + const { rows } = await sqlQuery(stranger.token, "SELECT * FROM published_qa"); + return rows.some(r => r.question === question); +} + +describe.skipIf(!canRun)("published_qa visibility", () => { + let scope: CleanupScope; + beforeEach(() => { + scope = createCleanupScope(); + }); + afterEach(async () => { + await scope.cleanup(); + }); + + it("does not expose rows added with publish:false", async ({ expect }) => { + const reviewer = await mintIdentity(); + scope.trackIdentity(reviewer.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewer.identity }, + }); + expect(enroll.status).toBe(200); + + const marker = uniqueMarker("unpublished"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "x", publish: false }, + }); + expect(add.status).toBe(200); + + expect(await publishedQaContains(marker)).toBe(false); + }); + + it("removes a row from published_qa when update-qa-entry sets publish:false", async ({ expect }) => { + const reviewer = await mintIdentity(); + scope.trackIdentity(reviewer.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewer.identity }, + }); + expect(enroll.status).toBe(200); + + const marker = uniqueMarker("to-unpublish"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "x", publish: true }, + }); + expect(add.status).toBe(200); + expect(await publishedQaContains(marker)).toBe(true); + + const qaId = await findManualQaEntryIdByQuestion(reviewer.token, marker); + expect(qaId).toBeDefined(); + + const update = await niceBackendFetch("/api/latest/internal/mcp-review/update-qa-entry", { + method: "POST", + accessType: "client", + body: { + qaId: qaId!.toString(), + question: marker, + answer: "x", + publish: false, + }, + }); + expect(update.status).toBe(200); + + expect(await publishedQaContains(marker)).toBe(false); + }); + + it("removes a row from published_qa when deleted", async ({ expect }) => { + const reviewer = await mintIdentity(); + scope.trackIdentity(reviewer.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewer.identity }, + }); + expect(enroll.status).toBe(200); + + const marker = uniqueMarker("to-delete"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "x", publish: true }, + }); + expect(add.status).toBe(200); + expect(await publishedQaContains(marker)).toBe(true); + + const qaId = await findManualQaEntryIdByQuestion(reviewer.token, marker); + expect(qaId).toBeDefined(); + + const del = await niceBackendFetch("/api/latest/internal/mcp-review/delete", { + method: "POST", + accessType: "client", + body: { qaId: qaId!.toString() }, + }); + expect(del.status).toBe(200); + + expect(await publishedQaContains(marker)).toBe(false); + }); + + it("lets reviewer B delete a row published by reviewer A (cross-reviewer integrity)", async ({ expect }) => { + // A publishes. + const reviewerA = await mintIdentity(); + scope.trackIdentity(reviewerA.identity); + await AiChatReviewer.createReviewer(); + const enrollA = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewerA.identity }, + }); + expect(enrollA.status).toBe(200); + + const marker = uniqueMarker("cross-reviewer"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "x", publish: true }, + }); + expect(add.status).toBe(200); + expect(await publishedQaContains(marker)).toBe(true); + + const qaId = await findManualQaEntryIdByQuestion(reviewerA.token, marker); + expect(qaId).toBeDefined(); + + // B deletes. fastSignUp re-points backendContext.userAuth to B; subsequent calls use B's auth. + const reviewerB = await mintIdentity(); + scope.trackIdentity(reviewerB.identity); + await AiChatReviewer.createReviewer(); + const enrollB = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewerB.identity }, + }); + expect(enrollB.status).toBe(200); + + const del = await niceBackendFetch("/api/latest/internal/mcp-review/delete", { + method: "POST", + accessType: "client", + body: { qaId: qaId!.toString() }, + }); + expect(del.status).toBe(200); + + expect(await publishedQaContains(marker)).toBe(false); + }); +}); diff --git a/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts b/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts new file mode 100644 index 0000000000..4d9f8ebeed --- /dev/null +++ b/apps/e2e/tests/spacetimedb/qa-entries-invariants.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { + createCleanupScope, + findManualQaEntryIdByQuestion, + isSpacetimedbReachable, + mintIdentity, + sqlQuery, + type CleanupScope, +} from "./helpers"; + +const canRun = await isSpacetimedbReachable(); + +function uniqueMarker(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +async function readManualEntry(token: string, qaId: bigint) { + const { rows } = await sqlQuery(token, "SELECT * FROM my_visible_qa_entries"); + const match = rows.find(r => { + const raw = r.id; + const id = typeof raw === "string" ? BigInt(raw) : typeof raw === "number" ? BigInt(raw) : raw; + return id === qaId; + }); + if (!match) throw new Error(`qa_entries row ${qaId.toString()} not found`); + return match as Record; +} + +function readOptional(value: unknown): T | undefined { + if (value == null) return undefined; + if (typeof value === "object" && value != null && "some" in value) { + return (value as { some: T }).some; + } + if (typeof value === "object" && value != null && "none" in value) return undefined; + return value as T; +} + +describe.skipIf(!canRun)("qa_entries CRUD invariants", () => { + let scope: CleanupScope; + beforeEach(() => { scope = createCleanupScope(); }); + afterEach(async () => { await scope.cleanup(); }); + + it("firstPublishedAt is immutable; lastPublishedAt updates per republish; both survive unpublish", async ({ expect }) => { + const reviewer = await mintIdentity(); + scope.trackIdentity(reviewer.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewer.identity }, + }); + expect(enroll.status).toBe(200); + + const marker = uniqueMarker("publish-history"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "a", publish: true }, + }); + expect(add.status).toBe(200); + + const qaId = await findManualQaEntryIdByQuestion(reviewer.token, marker); + expect(qaId).toBeDefined(); + + const initial = await readManualEntry(reviewer.token, qaId!); + const firstPublishedAt = readOptional(initial.first_published_at ?? initial.firstPublishedAt); + const lastPublishedAt1 = readOptional(initial.last_published_at ?? initial.lastPublishedAt); + expect(firstPublishedAt).toBeDefined(); + expect(lastPublishedAt1).toBeDefined(); + + // Unpublish: both timestamps must survive. + const unpub = await niceBackendFetch("/api/latest/internal/mcp-review/update-qa-entry", { + method: "POST", + accessType: "client", + body: { qaId: qaId!.toString(), question: marker, answer: "a", publish: false }, + }); + expect(unpub.status).toBe(200); + const afterUnpub = await readManualEntry(reviewer.token, qaId!); + expect(readOptional(afterUnpub.first_published_at ?? afterUnpub.firstPublishedAt)).toEqual(firstPublishedAt); + expect(readOptional(afterUnpub.last_published_at ?? afterUnpub.lastPublishedAt)).toEqual(lastPublishedAt1); + + // Republish: firstPublishedAt unchanged; lastPublishedAt advances. + await new Promise(r => setTimeout(r, 10)); + const rep = await niceBackendFetch("/api/latest/internal/mcp-review/update-qa-entry", { + method: "POST", + accessType: "client", + body: { qaId: qaId!.toString(), question: marker, answer: "a", publish: true }, + }); + expect(rep.status).toBe(200); + const afterRep = await readManualEntry(reviewer.token, qaId!); + expect(readOptional(afterRep.first_published_at ?? afterRep.firstPublishedAt)).toEqual(firstPublishedAt); + const lastPublishedAt2 = readOptional(afterRep.last_published_at ?? afterRep.lastPublishedAt); + expect(lastPublishedAt2).toBeDefined(); + expect(lastPublishedAt2).not.toEqual(lastPublishedAt1); + }); + + it("delete_qa_entry preserves the originating mcp_call_log row when present", async ({ expect }) => { + // Manual entries have no mcp_call_log row; this test asserts the inverse — that the + // delete reducer's scope is qa_entries only. Since manual entries have nothing to + // preserve in mcp_call_log, the test asserts via the row-count of qa_entries dropping + // by exactly 1, with no side effect on other tables. + const reviewer = await mintIdentity(); + scope.trackIdentity(reviewer.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewer.identity }, + }); + expect(enroll.status).toBe(200); + + const marker = uniqueMarker("delete-scope"); + scope.trackMcpQuestion(marker); + const add = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: marker, answer: "a", publish: true }, + }); + expect(add.status).toBe(200); + + const qaId = await findManualQaEntryIdByQuestion(reviewer.token, marker); + expect(qaId).toBeDefined(); + + const beforeQa = (await sqlQuery(reviewer.token, "SELECT * FROM my_visible_qa_entries")).rows.length; + const beforeMcp = (await sqlQuery(reviewer.token, "SELECT * FROM my_visible_mcp_call_log")).rows.length; + + const del = await niceBackendFetch("/api/latest/internal/mcp-review/delete", { + method: "POST", + accessType: "client", + body: { qaId: qaId!.toString() }, + }); + expect(del.status).toBe(200); + + const afterQa = (await sqlQuery(reviewer.token, "SELECT * FROM my_visible_qa_entries")).rows.length; + const afterMcp = (await sqlQuery(reviewer.token, "SELECT * FROM my_visible_mcp_call_log")).rows.length; + expect(afterQa).toBe(beforeQa - 1); + expect(afterMcp).toBe(beforeMcp); + }); +}); diff --git a/apps/e2e/tests/spacetimedb/reducer-auth.test.ts b/apps/e2e/tests/spacetimedb/reducer-auth.test.ts new file mode 100644 index 0000000000..0e619ac51a --- /dev/null +++ b/apps/e2e/tests/spacetimedb/reducer-auth.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe } from "vitest"; +import { it } from "../helpers"; +import { AiChatReviewer, niceBackendFetch } from "../backend/backend-helpers"; +import { callReducer, createCleanupScope, getSpacetimedbConfig, isSpacetimedbReachable, mintIdentity, opt, sqlQuery, type CleanupScope } from "./helpers"; + +const canRun = await isSpacetimedbReachable(); +const { logToken } = getSpacetimedbConfig(); + +function uniqueMarker(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +describe.skipIf(!canRun)("SpacetimeDB reducer auth", () => { + let scope: CleanupScope; + beforeEach(() => { + scope = createCleanupScope(); + }); + afterEach(async () => { + await scope.cleanup(); + }); + + it("a freshly-minted non-operator identity sees zero rows in my_visible_mcp_call_log", async ({ expect }) => { + // Seed a published row so the underlying mcp_call_log is definitely non-empty — + // otherwise a 0-row result could be a false positive from an empty table. + const reviewerIdentity = await mintIdentity(); + scope.trackIdentity(reviewerIdentity.identity); + await AiChatReviewer.createReviewer(); + const enroll = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: reviewerIdentity.identity }, + }); + expect(enroll.status).toBe(200); + const seedMarker = uniqueMarker("reducer-auth-seed"); + scope.trackMcpQuestion(seedMarker); + const seedPublish = await niceBackendFetch("/api/latest/internal/mcp-review/add-manual", { + method: "POST", + accessType: "client", + body: { question: seedMarker, answer: "a", publish: false }, + }); + expect(seedPublish.status).toBe(200); + + const stranger = await mintIdentity(); + const result = await sqlQuery(stranger.token, "SELECT * FROM my_visible_mcp_call_log"); + expect(result.rows.length).toBe(0); + }); + + // Smoke-test every mutating reducer's token gate. The existing add_operator test + // above catches regressions on that one reducer specifically; this loop ensures + // no new mutating reducer ships without a token check. If a reducer is added, it + // must be added here too or this smoke covers one less attack surface. + it("every mutating reducer rejects calls with a wrong log token", async ({ expect }) => { + const caller = await mintIdentity(); + const victim = await mintIdentity(); + const wrong = "definitely-not-the-real-token"; + const hexId = `0x${victim.identity}`; + + const cases = [ + { name: "add_operator", args: [wrong, [hexId], "some-user", "Some Name"] }, + { name: "remove_operator", args: [wrong, [hexId]] }, + { name: "enroll_service", args: [wrong, "Some Service"] }, + { name: "mark_human_reviewed", args: [wrong, "corr", "reviewer"] }, + { name: "unmark_human_reviewed", args: [wrong, "corr"] }, + { + name: "update_human_correction", + args: [wrong, "corr", "q", "a", false, "reviewer"], + }, + { name: "add_manual_qa", args: [wrong, "q", "a", false, "reviewer"] }, + { name: "delete_qa_entry", args: [wrong, "corr"] }, + { + name: "log_mcp_call", + args: [wrong, "corr", opt(null), "tool", "reason", "prompt", "q", "r", 0, "[]", 0n, "model", opt(null)], + }, + { + name: "update_mcp_qa_review", + args: [wrong, "corr", false, false, false, "[]", "", 0, "model", opt(null), opt(null)], + }, + { + name: "log_ai_query", + args: [wrong, "corr", "chat", "sys", "q", "s", "model", false, opt(null), opt(null), "[]", "[]", "[]", "text", opt(null), opt(null), opt(null), opt(null), 0, 0n, opt(null), opt(null), opt(null)], + }, + ]; + + for (const { name, args } of cases) { + const result = await callReducer(caller.token, name, args); + expect(result.ok, `reducer ${name} should reject wrong token`).toBe(false); + expect(result.body, `reducer ${name} should report invalid-token error`).toContain("Invalid log token"); + } + }); + + it.skipIf(!logToken)( + "rejects add_operator when an existing identity is claimed under a different stackUserId", + async ({ expect }) => { + // Enroll identity X with stackUserId=A via the backend endpoint (legitimate flow). + const target = await mintIdentity(); + scope.trackIdentity(target.identity); + const callerA = await mintIdentity(); + await AiChatReviewer.createReviewer(); + const enrollA = await niceBackendFetch("/api/latest/internal/spacetimedb-enroll-reviewer", { + method: "POST", + accessType: "client", + body: { identity: target.identity }, + }); + expect(enrollA.status).toBe(200); + + // Now directly call add_operator with a DIFFERENT stackUserId for the same identity. + // Simulates an attacker with the log token trying to relabel X's row. + const result = await callReducer(callerA.token, "add_operator", [ + logToken!, + [`0x${target.identity}`], + "attacker-different-stack-user-id", + "Attacker Display Name", + ]); + + expect(result.ok).toBe(false); + expect(result.body).toContain("Identity is bound to a different Stack user"); + }, + ); +}); diff --git a/apps/internal-tool/scripts/pre-dev.mjs b/apps/internal-tool/scripts/pre-dev.mjs index ca6beb5123..e9374c21cb 100644 --- a/apps/internal-tool/scripts/pre-dev.mjs +++ b/apps/internal-tool/scripts/pre-dev.mjs @@ -1,10 +1,13 @@ #!/usr/bin/env node // Runs before `next dev`. Publishes the SpacetimeDB module to the local server -// if the spacetime CLI is installed. Otherwise, warns and continues so the -// dev server still starts (useful in CI and for contributors who haven't -// installed the CLI yet). +// if the spacetime CLI is installed, then provisions a service identity token +// for the backend if one isn't already set (or is stale). Otherwise, warns and +// continues so the dev server still starts (useful in CI and for contributors +// who haven't installed the CLI yet). import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync, writeFileSync, appendFileSync } from "node:fs"; +import { resolve } from "node:path"; const which = spawnSync(process.platform === "win32" ? "where" : "which", ["spacetime"], { stdio: "ignore", @@ -21,9 +24,113 @@ const publish = spawnSync("pnpm", ["spacetime:publish:local"], { }); if (publish.status !== 0) { - console.warn( - "\n[internal-tool] spacetime publish to local failed (is `spacetime start` running?). Skipping; starting Next anyway.\n", + console.warn(`[internal-tool] spacetime publish failed (status ${publish.status}); exiting`); + process.exit(publish.status ?? 1); +} + +await provisionServiceToken(); +await runQaEntriesBackfill(); + +async function runQaEntriesBackfill() { + const dbName = process.env.STACK_SPACETIMEDB_DB_NAME ?? "stack-auth-llm"; + const logToken = process.env.STACK_MCP_LOG_TOKEN ?? ""; + if (!logToken) { + console.warn("[internal-tool] STACK_MCP_LOG_TOKEN not set; skipping qa_entries backfill."); + return; + } + const result = spawnSync( + "spacetime", + ["call", dbName, "backfill_qa_entries", JSON.stringify(logToken)], + { stdio: "inherit" }, + ); + if (result.status !== 0) { + console.warn(`[internal-tool] backfill_qa_entries returned ${result.status}; ignoring (may already be migrated).`); + } +} + +async function provisionServiceToken() { + const portPrefix = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX ?? "81"; + const spacetimeHttpUrl = `http://127.0.0.1:${portPrefix}39`; + const dbName = process.env.STACK_SPACETIMEDB_DB_NAME ?? "stack-auth-llm"; + const backendEnvLocal = resolve("../backend/.env.development.local"); + const backendEnvDev = resolve("../backend/.env.development"); + + + const existingToken = + readEnvVar(backendEnvLocal, "STACK_SPACETIMEDB_SERVICE_TOKEN") || + readEnvVar(backendEnvDev, "STACK_SPACETIMEDB_SERVICE_TOKEN"); + + if (existingToken) { + const stillValid = await probeToken(spacetimeHttpUrl, dbName, existingToken); + if (stillValid) { + return; + } + console.log("[internal-tool] Existing STACK_SPACETIMEDB_SERVICE_TOKEN is stale; re-minting..."); + removeEnvVar(backendEnvLocal, "STACK_SPACETIMEDB_SERVICE_TOKEN"); + } else { + console.log("[internal-tool] Minting SpacetimeDB service token for backend..."); + } + + let token; + try { + const res = await fetch(`${spacetimeHttpUrl}/v1/identity`, { method: "POST" }); + if (!res.ok) { + console.warn(`[internal-tool] Failed to mint service token: HTTP ${res.status}. Backend SpacetimeDB features will error until STACK_SPACETIMEDB_SERVICE_TOKEN is set manually.`); + return; + } + const body = await res.json(); + token = body.token; + } catch (err) { + console.warn(`[internal-tool] Failed to mint service token: ${err.message}. Backend SpacetimeDB features will error until STACK_SPACETIMEDB_SERVICE_TOKEN is set manually.`); + return; + } + + if (typeof token !== "string" || token.trim() === "") { + console.warn("[internal-tool] /v1/identity returned no usable token field; skipping write to .env.development.local. Backend SpacetimeDB features will error until STACK_SPACETIMEDB_SERVICE_TOKEN is set manually."); + return; + } + + const existingContent = existsSync(backendEnvLocal) ? readFileSync(backendEnvLocal, "utf8") : ""; + const prefix = existingContent && !existingContent.endsWith("\n") ? "\n" : ""; + appendFileSync( + backendEnvLocal, + `${prefix}# Auto-provisioned by apps/internal-tool/scripts/pre-dev.mjs\nSTACK_SPACETIMEDB_SERVICE_TOKEN=${token}\n`, ); + console.log(`[internal-tool] Wrote STACK_SPACETIMEDB_SERVICE_TOKEN to ${backendEnvLocal}`); + console.log("[internal-tool] Restart the backend dev server if already running to pick up the new env var."); +} + +async function probeToken(spacetimeHttpUrl, dbName, token) { + try { + const res = await fetch(`${spacetimeHttpUrl}/v1/database/${encodeURIComponent(dbName)}/sql`, { + method: "POST", + headers: { "Authorization": `Bearer ${token}` }, + body: "SELECT 1", + }); + if (res.status === 401) return false; + if (res.ok) return true; + return true; + } catch { + return true; + } } -process.exit(0); +function readEnvVar(filePath, key) { + if (!existsSync(filePath)) return null; + const content = readFileSync(filePath, "utf8"); + const match = content.match(new RegExp(`^${key}=(.+)$`, "m")); + if (!match) return null; + const value = match[1].trim(); + return value === "" ? null : value; +} + +function removeEnvVar(filePath, key) { + if (!existsSync(filePath)) return; + const content = readFileSync(filePath, "utf8"); + const pattern = new RegExp( + `(^# Auto-provisioned by apps/internal-tool/scripts/pre-dev\\.mjs\\n)?^${key}=.*\\n?`, + "m", + ); + const updated = content.replace(pattern, ""); + writeFileSync(filePath, updated, "utf8"); +} diff --git a/apps/internal-tool/spacetimedb/src/index.ts b/apps/internal-tool/spacetimedb/src/index.ts index da052f4bf7..d90fafbc38 100644 --- a/apps/internal-tool/spacetimedb/src/index.ts +++ b/apps/internal-tool/spacetimedb/src/index.ts @@ -1,14 +1,16 @@ import { schema, t, table, SenderError } from 'spacetimedb/server'; +import type { Timestamp } from 'spacetimedb'; // Injected at publish time by the spacetime:inject-token pnpm script from STACK_MCP_LOG_TOKEN env var. // Must match STACK_MCP_LOG_TOKEN in the backend .env. const EXPECTED_LOG_TOKEN = '__SPACETIMEDB_LOG_TOKEN__'; const mcpCallLog = table( - { name: 'mcp_call_log', public: true }, + { name: 'mcp_call_log', public: false }, { id: t.u64().primaryKey().autoInc(), - correlationId: t.string(), + shard: t.u8().index('btree'), + correlationId: t.string().unique(), conversationId: t.string().optional(), createdAt: t.timestamp(), toolName: t.string(), @@ -21,7 +23,6 @@ const mcpCallLog = table( durationMs: t.u64(), modelId: t.string(), errorMessage: t.string().optional(), - // QA review fields (populated asynchronously after initial log) qaReviewedAt: t.timestamp().optional(), qaNeedsHumanReview: t.bool().optional(), qaAnswerCorrect: t.bool().optional(), @@ -32,19 +33,217 @@ const mcpCallLog = table( qaReviewModelId: t.string().optional(), qaConversationJson: t.string().optional(), qaErrorMessage: t.string().optional(), - // Human review humanReviewedAt: t.timestamp().optional(), humanReviewedBy: t.string().optional(), - // Human corrections & publishing humanCorrectedQuestion: t.string().optional(), humanCorrectedAnswer: t.string().optional(), - publishedToQa: t.bool().optional(), + publishedToQa: t.bool().index('btree'), publishedAt: t.timestamp().optional(), } ); -const spacetimedb = schema({ mcpCallLog }); +const aiQueryLog = table( + { name: 'ai_query_log', public: false }, + { + id: t.u64().primaryKey().autoInc(), + shard: t.u8().index('btree'), + correlationId: t.string().unique(), + createdAt: t.timestamp(), + mode: t.string(), + systemPromptId: t.string(), + quality: t.string(), + speed: t.string(), + modelId: t.string(), + isAuthenticated: t.bool(), + projectId: t.string().optional(), + userId: t.string().optional(), + requestedToolsJson: t.string(), + messagesJson: t.string(), + stepsJson: t.string(), + finalText: t.string(), + inputTokens: t.u32().optional(), + outputTokens: t.u32().optional(), + cachedInputTokens: t.u32().optional(), + cacheCreationTokens: t.u32().optional(), + costUsd: t.f64().optional(), + cacheDiscountUsd: t.f64().optional(), + openrouterGenerationId: t.string().optional(), + stepCount: t.u32(), + durationMs: t.u64(), + errorMessage: t.string().optional(), + conversationId: t.string().optional(), + } +); + +const operators = table( + { name: 'operators', public: true }, + { + identity: t.identity().primaryKey(), + addedAt: t.timestamp(), + stackUserId: t.string(), + displayName: t.string(), + } +); + +const qaEntries = table( + { name: 'qa_entries', public: false }, + { + id: t.u64().primaryKey().autoInc(), + shard: t.u8().index('btree'), + sourceMcpCorrelationId: t.string().optional(), + question: t.string(), + answer: t.string(), + createdBy: t.string(), + createdAt: t.timestamp(), + lastEditedBy: t.string(), + lastEditedAt: t.timestamp(), + published: t.bool().index('btree'), + firstPublishedAt: t.timestamp().optional(), + lastPublishedAt: t.timestamp().optional(), + } +); + +const spacetimedb = schema({ mcpCallLog, aiQueryLog, operators, qaEntries }); export default spacetimedb; +export const operatorsVisibility = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM operators WHERE identity = :sender' +); +export const myVisibleMcpCallLog = spacetimedb.view( + { name: 'my_visible_mcp_call_log', public: true }, + t.array(mcpCallLog.rowType), + (ctx) => { + if (ctx.db.operators.identity.find(ctx.sender) == null) return []; + return Array.from(ctx.db.mcpCallLog.shard.filter(0)); + } +); +export const myVisibleAiQueryLog = spacetimedb.view( + { name: 'my_visible_ai_query_log', public: true }, + t.array(aiQueryLog.rowType), + (ctx) => { + if (ctx.db.operators.identity.find(ctx.sender) == null) return []; + return Array.from(ctx.db.aiQueryLog.shard.filter(0)); + } +); +export const myVisibleQaEntries = spacetimedb.view( + { name: 'my_visible_qa_entries', public: true }, + t.array(qaEntries.rowType), + (ctx) => { + if (ctx.db.operators.identity.find(ctx.sender) == null) return []; + return Array.from(ctx.db.qaEntries.shard.filter(0)); + } +); + +// Public view for the /questions page — returns rows reviewers have explicitly +// published. Uses `anonymousView` so SpacetimeDB materializes once and shares +// the result across all subscribers. Projected to only fields the public page +// needs; everything else (reviewer attribution, QA internals, raw prompt, +// tool-call metadata) stays private. +const publishedQaRow = t.object('PublishedQaRow', { + id: t.u64(), + question: t.string(), + answer: t.string(), + publishedAt: t.timestamp().optional(), +}); + +export const publishedQa = spacetimedb.anonymousView( + { name: 'published_qa', public: true }, + t.array(publishedQaRow), + (ctx) => { + const out: Array<{ + id: bigint, + question: string, + answer: string, + publishedAt: Timestamp | undefined, + }> = []; + for (const row of ctx.db.qaEntries.published.filter(true)) { + out.push({ + id: row.id, + question: row.question, + answer: row.answer, + publishedAt: row.lastPublishedAt, + }); + } + return out; + }, +); + +export const add_operator = spacetimedb.reducer( + { + token: t.string(), + identity: t.identity(), + stackUserId: t.string(), + displayName: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + if (/^__.*__$/.test(args.stackUserId)) { + throw new SenderError('stackUserId pattern __*__ is reserved'); + } + const existing = ctx.db.operators.identity.find(args.identity); + if (existing != null) { + if (existing.stackUserId !== args.stackUserId) { + throw new SenderError('Identity is bound to a different Stack user'); + } + ctx.db.operators.identity.update({ + identity: args.identity, + addedAt: existing.addedAt, + stackUserId: args.stackUserId, + displayName: args.displayName, + }); + return; + } + const stale = []; + for (const row of ctx.db.operators.iter()) { + if (row.stackUserId === args.stackUserId) { + stale.push(row); + } + } + for (const row of stale) { + ctx.db.operators.identity.delete(row.identity); + } + ctx.db.operators.insert({ + identity: args.identity, + addedAt: ctx.timestamp, + stackUserId: args.stackUserId, + displayName: args.displayName, + }); + } +); + +export const remove_operator = spacetimedb.reducer( + { + token: t.string(), + identity: t.identity(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.operators.identity.delete(args.identity); + } +); + +export const enroll_service = spacetimedb.reducer( + { + token: t.string(), + displayName: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const existing = ctx.db.operators.identity.find(ctx.sender); + if (existing != null) return; + ctx.db.operators.insert({ + identity: ctx.sender, + addedAt: ctx.timestamp, + stackUserId: '__service__', + displayName: args.displayName, + }); + } +); export const log_mcp_call = spacetimedb.reducer( { @@ -68,6 +267,7 @@ export const log_mcp_call = spacetimedb.reducer( } ctx.db.mcpCallLog.insert({ id: 0n, + shard: 0, correlationId: args.correlationId, conversationId: args.conversationId, createdAt: ctx.timestamp, @@ -81,6 +281,7 @@ export const log_mcp_call = spacetimedb.reducer( durationMs: args.durationMs, modelId: args.modelId, errorMessage: args.errorMessage, + publishedToQa: false, } as Parameters[0]); } ); @@ -103,26 +304,23 @@ export const update_mcp_qa_review = spacetimedb.reducer( if (args.token !== EXPECTED_LOG_TOKEN) { throw new SenderError('Invalid log token'); } - for (const row of ctx.db.mcpCallLog.iter()) { - if (row.correlationId === args.correlationId) { - ctx.db.mcpCallLog.delete(row); - ctx.db.mcpCallLog.insert({ - ...row, - qaReviewedAt: ctx.timestamp, - qaNeedsHumanReview: args.qaNeedsHumanReview, - qaAnswerCorrect: args.qaAnswerCorrect, - qaAnswerRelevant: args.qaAnswerRelevant, - qaFlagsJson: args.qaFlagsJson, - qaImprovementSuggestions: args.qaImprovementSuggestions, - qaOverallScore: args.qaOverallScore, - qaReviewModelId: args.qaReviewModelId, - qaConversationJson: args.qaConversationJson, - qaErrorMessage: args.qaErrorMessage, - }); - return; - } + const row = ctx.db.mcpCallLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); } - throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + ctx.db.mcpCallLog.id.update({ + ...row, + qaReviewedAt: ctx.timestamp, + qaNeedsHumanReview: args.qaNeedsHumanReview, + qaAnswerCorrect: args.qaAnswerCorrect, + qaAnswerRelevant: args.qaAnswerRelevant, + qaFlagsJson: args.qaFlagsJson, + qaImprovementSuggestions: args.qaImprovementSuggestions, + qaOverallScore: args.qaOverallScore, + qaReviewModelId: args.qaReviewModelId, + qaConversationJson: args.qaConversationJson, + qaErrorMessage: args.qaErrorMessage, + }); } ); @@ -136,50 +334,89 @@ export const mark_human_reviewed = spacetimedb.reducer( if (args.token !== EXPECTED_LOG_TOKEN) { throw new SenderError('Invalid log token'); } - for (const row of ctx.db.mcpCallLog.iter()) { - if (row.correlationId === args.correlationId) { - ctx.db.mcpCallLog.delete(row); - ctx.db.mcpCallLog.insert({ - ...row, - humanReviewedAt: ctx.timestamp, - humanReviewedBy: args.reviewedBy, - }); - return; - } + const row = ctx.db.mcpCallLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } + ctx.db.mcpCallLog.id.update({ + ...row, + humanReviewedAt: ctx.timestamp, + humanReviewedBy: args.reviewedBy, + }); + } +); + +export const unmark_human_reviewed = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.mcpCallLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); } - throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + ctx.db.mcpCallLog.id.update({ + ...row, + humanReviewedAt: undefined, + humanReviewedBy: undefined, + }); } ); -export const update_human_correction = spacetimedb.reducer( +export const upsert_qa_from_call = spacetimedb.reducer( { token: t.string(), correlationId: t.string(), - correctedQuestion: t.string(), - correctedAnswer: t.string(), + question: t.string(), + answer: t.string(), publish: t.bool(), - reviewedBy: t.string(), + editedBy: t.string(), }, (ctx, args) => { if (args.token !== EXPECTED_LOG_TOKEN) { throw new SenderError('Invalid log token'); } - for (const row of ctx.db.mcpCallLog.iter()) { - if (row.correlationId === args.correlationId) { - ctx.db.mcpCallLog.delete(row); - ctx.db.mcpCallLog.insert({ - ...row, - humanCorrectedQuestion: args.correctedQuestion, - humanCorrectedAnswer: args.correctedAnswer, - humanReviewedAt: row.humanReviewedAt ?? ctx.timestamp, - humanReviewedBy: row.humanReviewedBy ?? args.reviewedBy, - publishedToQa: args.publish, - publishedAt: args.publish ? (row.publishedAt ?? ctx.timestamp) : undefined, - }); - return; + if (ctx.db.mcpCallLog.correlationId.find(args.correlationId) == null) { + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } + let existing = null; + for (const row of ctx.db.qaEntries.shard.filter(0)) { + if (row.sourceMcpCorrelationId === args.correlationId) { + existing = row; + break; } } - throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + if (existing != null) { + ctx.db.qaEntries.id.update({ + ...existing, + question: args.question, + answer: args.answer, + lastEditedBy: args.editedBy, + lastEditedAt: ctx.timestamp, + published: args.publish, + firstPublishedAt: args.publish ? (existing.firstPublishedAt ?? ctx.timestamp) : existing.firstPublishedAt, + lastPublishedAt: args.publish ? ctx.timestamp : existing.lastPublishedAt, + }); + return; + } + ctx.db.qaEntries.insert({ + id: 0n, + shard: 0, + sourceMcpCorrelationId: args.correlationId, + question: args.question, + answer: args.answer, + createdBy: args.editedBy, + createdAt: ctx.timestamp, + lastEditedBy: args.editedBy, + lastEditedAt: ctx.timestamp, + published: args.publish, + firstPublishedAt: args.publish ? ctx.timestamp : undefined, + lastPublishedAt: args.publish ? ctx.timestamp : undefined, + } as Parameters[0]); } ); @@ -189,51 +426,290 @@ export const add_manual_qa = spacetimedb.reducer( question: t.string(), answer: t.string(), publish: t.bool(), - reviewedBy: t.string(), + createdBy: t.string(), }, (ctx, args) => { if (args.token !== EXPECTED_LOG_TOKEN) { throw new SenderError('Invalid log token'); } - ctx.db.mcpCallLog.insert({ + ctx.db.qaEntries.insert({ id: 0n, - correlationId: ctx.newUuidV4().toString(), - createdAt: ctx.timestamp, - toolName: "manual", - reason: "Manually added Q&A", - userPrompt: "", + shard: 0, + sourceMcpCorrelationId: undefined, question: args.question, - response: "", - stepCount: 0, - innerToolCallsJson: "[]", - durationMs: 0n, - modelId: "human", - humanCorrectedQuestion: args.question, - humanCorrectedAnswer: args.answer, - humanReviewedAt: ctx.timestamp, - humanReviewedBy: args.reviewedBy, - publishedToQa: args.publish, - publishedAt: args.publish ? ctx.timestamp : undefined, - } as Parameters[0]); + answer: args.answer, + createdBy: args.createdBy, + createdAt: ctx.timestamp, + lastEditedBy: args.createdBy, + lastEditedAt: ctx.timestamp, + published: args.publish, + firstPublishedAt: args.publish ? ctx.timestamp : undefined, + lastPublishedAt: args.publish ? ctx.timestamp : undefined, + } as Parameters[0]); } ); export const delete_qa_entry = spacetimedb.reducer( { token: t.string(), - correlationId: t.string(), + qaId: t.u64(), }, (ctx, args) => { if (args.token !== EXPECTED_LOG_TOKEN) { throw new SenderError('Invalid log token'); } - for (const row of ctx.db.mcpCallLog.iter()) { - if (row.correlationId === args.correlationId) { - ctx.db.mcpCallLog.delete(row); - return; + const row = ctx.db.qaEntries.id.find(args.qaId); + if (row == null) { + throw new SenderError('QA entry not found for qaId: ' + args.qaId.toString()); + } + ctx.db.qaEntries.id.delete(row.id); + } +); + +// Pure publish state transition. Same firstPublishedAt / lastPublishedAt +// semantics as upsert_qa_from_call. +export const set_qa_published = spacetimedb.reducer( + { + token: t.string(), + qaId: t.u64(), + publish: t.bool(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.qaEntries.id.find(args.qaId); + if (row == null) { + throw new SenderError('QA entry not found for qaId: ' + args.qaId.toString()); + } + ctx.db.qaEntries.id.update({ + ...row, + published: args.publish, + firstPublishedAt: args.publish ? (row.firstPublishedAt ?? ctx.timestamp) : row.firstPublishedAt, + lastPublishedAt: args.publish ? ctx.timestamp : row.lastPublishedAt, + }); + } +); + +export const update_qa_entry_with_publish = spacetimedb.reducer( + { + token: t.string(), + qaId: t.u64(), + question: t.string(), + answer: t.string(), + publish: t.bool(), + editedBy: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.qaEntries.id.find(args.qaId); + if (row == null) { + throw new SenderError('QA entry not found for qaId: ' + args.qaId.toString()); + } + ctx.db.qaEntries.id.update({ + ...row, + question: args.question, + answer: args.answer, + lastEditedBy: args.editedBy, + lastEditedAt: ctx.timestamp, + published: args.publish, + firstPublishedAt: args.publish ? (row.firstPublishedAt ?? ctx.timestamp) : row.firstPublishedAt, + lastPublishedAt: args.publish ? ctx.timestamp : row.lastPublishedAt, + }); + } +); + +// Text-only edit. Doesn't touch publish state. +export const update_qa_entry = spacetimedb.reducer( + { + token: t.string(), + qaId: t.u64(), + question: t.string(), + answer: t.string(), + editedBy: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.qaEntries.id.find(args.qaId); + if (row == null) { + throw new SenderError('QA entry not found for qaId: ' + args.qaId.toString()); + } + ctx.db.qaEntries.id.update({ + ...row, + question: args.question, + answer: args.answer, + lastEditedBy: args.editedBy, + lastEditedAt: ctx.timestamp, + }); + } +); + +export const backfill_qa_entries = spacetimedb.reducer( + { + token: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + for (const call of ctx.db.mcpCallLog.shard.filter(0)) { + const hasEditorialContent = + call.humanCorrectedQuestion != null || + call.humanCorrectedAnswer != null || + call.publishedToQa === true || + call.toolName === 'manual'; + if (!hasEditorialContent) continue; + let alreadyMigrated = false; + for (const existing of ctx.db.qaEntries.shard.filter(0)) { + if (existing.sourceMcpCorrelationId === call.correlationId) { + alreadyMigrated = true; + break; + } } + if (alreadyMigrated) continue; + const reviewerId = call.humanReviewedBy ?? '__backfill__'; + const reviewedAt = call.humanReviewedAt ?? call.createdAt; + ctx.db.qaEntries.insert({ + id: 0n, + shard: 0, + sourceMcpCorrelationId: call.toolName === 'manual' ? undefined : call.correlationId, + question: call.humanCorrectedQuestion ?? call.question, + answer: call.humanCorrectedAnswer ?? call.response, + createdBy: reviewerId, + createdAt: reviewedAt, + lastEditedBy: reviewerId, + lastEditedAt: reviewedAt, + published: call.publishedToQa, + firstPublishedAt: call.publishedToQa ? (call.publishedAt ?? reviewedAt) : undefined, + lastPublishedAt: call.publishedToQa ? (call.publishedAt ?? reviewedAt) : undefined, + } as Parameters[0]); + } + } +); + + +export const log_ai_query = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + mode: t.string(), + systemPromptId: t.string(), + quality: t.string(), + speed: t.string(), + modelId: t.string(), + isAuthenticated: t.bool(), + projectId: t.string().optional(), + userId: t.string().optional(), + requestedToolsJson: t.string(), + messagesJson: t.string(), + stepsJson: t.string(), + finalText: t.string(), + inputTokens: t.u32().optional(), + outputTokens: t.u32().optional(), + cachedInputTokens: t.u32().optional(), + cacheCreationTokens: t.u32().optional(), + costUsd: t.f64().optional(), + cacheDiscountUsd: t.f64().optional(), + openrouterGenerationId: t.string().optional(), + stepCount: t.u32(), + durationMs: t.u64(), + errorMessage: t.string().optional(), + conversationId: t.string().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + ctx.db.aiQueryLog.insert({ + id: 0n, + shard: 0, + correlationId: args.correlationId, + createdAt: ctx.timestamp, + mode: args.mode, + systemPromptId: args.systemPromptId, + quality: args.quality, + speed: args.speed, + modelId: args.modelId, + isAuthenticated: args.isAuthenticated, + projectId: args.projectId, + userId: args.userId, + requestedToolsJson: args.requestedToolsJson, + messagesJson: args.messagesJson, + stepsJson: args.stepsJson, + finalText: args.finalText, + inputTokens: args.inputTokens, + outputTokens: args.outputTokens, + cachedInputTokens: args.cachedInputTokens, + cacheCreationTokens: args.cacheCreationTokens, + costUsd: args.costUsd, + cacheDiscountUsd: args.cacheDiscountUsd, + openrouterGenerationId: args.openrouterGenerationId, + stepCount: args.stepCount, + durationMs: args.durationMs, + errorMessage: args.errorMessage, + conversationId: args.conversationId, + } as Parameters[0]); + } +); + +export const update_ai_query_cost = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + costUsd: t.f64().optional(), + cacheDiscountUsd: t.f64().optional(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.aiQueryLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('AI query log not found for correlationId: ' + args.correlationId); + } + ctx.db.aiQueryLog.id.update({ + ...row, + costUsd: args.costUsd ?? row.costUsd, + cacheDiscountUsd: args.cacheDiscountUsd ?? row.cacheDiscountUsd, + }); + } +); + +export const delete_mcp_call_log = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.mcpCallLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + } + ctx.db.mcpCallLog.id.delete(row.id); + } +); + +export const delete_ai_query_log = spacetimedb.reducer( + { + token: t.string(), + correlationId: t.string(), + }, + (ctx, args) => { + if (args.token !== EXPECTED_LOG_TOKEN) { + throw new SenderError('Invalid log token'); + } + const row = ctx.db.aiQueryLog.correlationId.find(args.correlationId); + if (row == null) { + throw new SenderError('Log entry not found for correlationId: ' + args.correlationId); } - throw new SenderError('Call log not found for correlationId: ' + args.correlationId); + ctx.db.aiQueryLog.id.delete(row.id); } ); diff --git a/apps/internal-tool/src/app/app-client.tsx b/apps/internal-tool/src/app/app-client.tsx index d360c260ad..f09d5f0deb 100644 --- a/apps/internal-tool/src/app/app-client.tsx +++ b/apps/internal-tool/src/app/app-client.tsx @@ -1,23 +1,76 @@ import { useUser } from "@stackframe/stack"; import { clsx } from "clsx"; -import { useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { Identity } from "spacetimedb"; import { AddManualQa } from "../components/AddManualQa"; import { Analytics } from "../components/Analytics"; import { CallLogDetail } from "../components/CallLogDetail"; import { CallLogList } from "../components/CallLogList"; import { KnowledgeBase } from "../components/KnowledgeBase"; -import { useMcpCallLogs } from "../hooks/useSpacetimeDB"; -import { makeMcpReviewApi } from "../lib/mcp-review-api"; -import type { McpCallLogRow } from "../types"; +import { Usage } from "../components/Usage"; +import { UsageDetail } from "../components/UsageDetail"; +import { useAiQueryLogs, useMcpCallLogs, useQaEntries } from "../hooks/useSpacetimeDB"; +import { enrollSpacetimeReviewer, makeMcpReviewApi } from "../lib/mcp-review-api"; +import type { AiQueryLogRow, McpCallLogRow } from "../types"; -type Tab = "calls" | "knowledge" | "analytics"; +type Tab = "calls" | "knowledge" | "usage"; +const TAB_STORAGE_KEY = "internal-tool-active-tab"; +const VALID_TABS: readonly Tab[] = ["calls", "knowledge", "usage"]; + +function readInitialTab(): Tab { + // sessionStorage is per-tab: reload preserves the active tab, but a brand-new + // browser tab gets the default ("calls"). + if (typeof window === "undefined") return "calls"; + const saved = window.sessionStorage.getItem(TAB_STORAGE_KEY); + if (saved != null && (VALID_TABS as readonly string[]).includes(saved)) { + return saved as Tab; + } + return "calls"; +} export default function App() { const user = useUser({ or: process.env.NODE_ENV === "development" ? "redirect" : "return-null" }); const [selectedRow, setSelectedRow] = useState(null); + const [selectedUsageRow, setSelectedUsageRow] = useState(null); const [showAddQa, setShowAddQa] = useState(false); - const [tab, setTab] = useState("calls"); - const { rows, connectionState } = useMcpCallLogs(); + const [tab, setTab] = useState(readInitialTab); + + useEffect(() => { + if (typeof window === "undefined") return; + window.sessionStorage.setItem(TAB_STORAGE_KEY, tab); + }, [tab]); + const enrolledRef = useRef>>(new Map()); + const ensureEnrolled = useCallback(async (identity: Identity) => { + if (!user) throw new Error("Not authenticated"); + const key = identity.toHexString(); + const existing = enrolledRef.current.get(key); + if (existing) return await existing; + const promise = (async () => { + const { accessToken, refreshToken } = await user.getAuthJson(); + const authHeaders: Record = {}; + if (accessToken) authHeaders["x-stack-access-token"] = accessToken; + if (refreshToken) authHeaders["x-stack-refresh-token"] = refreshToken; + try { + await enrollSpacetimeReviewer({ identity: key }, authHeaders); + } catch (err) { + enrolledRef.current.delete(key); + throw err; + } + })(); + enrolledRef.current.set(key, promise); + return await promise; + }, [user]); + const isAiChatReviewer = Boolean( + (user?.clientReadOnlyMetadata as Record | null)?.isAiChatReviewer, + ); + const memoizedEnsureEnrolled = useMemo( + () => (user && isAiChatReviewer) ? ensureEnrolled : undefined, + [user, isAiChatReviewer, ensureEnrolled], + ); + + const { rows, connectionState } = useMcpCallLogs(memoizedEnsureEnrolled); + const { rows: usageRows, connectionState: usageConnectionState } = useAiQueryLogs(memoizedEnsureEnrolled); + const { rows: qaRows } = useQaEntries(memoizedEnsureEnrolled); if (!user) { return ( @@ -42,20 +95,17 @@ export default function App() { ); } - if (process.env.NODE_ENV !== "development") { - const metadata = user.clientReadOnlyMetadata as Record | null; - if (!metadata?.isAiChatReviewer) { - return ( -
-
-

Access Denied

-

- You are signed in as {user.displayName ?? user.primaryEmail}, but your account is not approved. -

-
+ if (!isAiChatReviewer) { + return ( +
+
+

Access Denied

+

+ You are signed in as {user.displayName ?? user.primaryEmail}, but your account is not approved. +

- ); - } +
+ ); } const currentSelectedRow = selectedRow @@ -73,11 +123,13 @@ export default function App() { } return ( -
-
-
+
+
+

MCP Review Tool

- {/* Tabs */} +
+ {/* Tabs — centered */} +
-
- +
+ {tab === "knowledge" && ( + + )} {user.displayName ?? user.primaryEmail}
@@ -138,61 +192,79 @@ export default function App() { /> )} - {tab === "calls" && ( -
-
- -
- {currentSelectedRow && ( - - )} -
- )} + + {currentSelectedRow && ( + + )} + + )} - {tab === "knowledge" && ( -
- { - getApi() - .then(api => api.updateCorrection({ correlationId, correctedQuestion: question, correctedAnswer: answer, publish })) - .catch(() => { /* errors are surfaced by UI state */ }); - }} - onDelete={(correlationId) => { - getApi() - .then(api => api.delete({ correlationId })) - .catch(() => { /* errors are surfaced by UI state */ }); - }} - /> -
- )} + {tab === "knowledge" && ( +
+
+ + getApi().then(api => api.updateQaEntry({ qaId: qaId.toString(), question, answer, publish })) + } + onDelete={(qaId) => + getApi().then(api => api.delete({ qaId: qaId.toString() })) + } + /> +
+
+ )} - {tab === "analytics" && ( -
- -
- )} + {tab === "usage" && ( + <> +
+
+ +
+
+ {selectedUsageRow && ( + + )} + + )} +
); } diff --git a/apps/internal-tool/src/app/questions/page.tsx b/apps/internal-tool/src/app/questions/page.tsx index 2fcadcc2d6..a9b8a38ef3 100644 --- a/apps/internal-tool/src/app/questions/page.tsx +++ b/apps/internal-tool/src/app/questions/page.tsx @@ -4,21 +4,19 @@ import { useMemo } from "react"; import Markdown from "react-markdown"; import remarkGfm from "remark-gfm"; import { format } from "date-fns"; -import { useMcpCallLogs } from "../../hooks/useSpacetimeDB"; +import { usePublishedQa } from "../../hooks/useSpacetimeDB"; import { toDate } from "../../utils"; import { markdownComponents } from "../../components/markdown-components"; export default function QuestionsPage() { - const { rows, connectionState } = useMcpCallLogs(); + const { rows, connectionState } = usePublishedQa(); const publishedQa = useMemo(() => { - return rows - .filter(r => r.publishedToQa) - .sort((a, b) => { - const aTime = a.publishedAt ? Number(toDate(a.publishedAt)) : 0; - const bTime = b.publishedAt ? Number(toDate(b.publishedAt)) : 0; - return bTime - aTime; - }); + return [...rows].sort((a, b) => { + const aTime = a.publishedAt ? Number(toDate(a.publishedAt)) : 0; + const bTime = b.publishedAt ? Number(toDate(b.publishedAt)) : 0; + return bTime - aTime; + }); }, [rows]); if (connectionState === "connecting") { @@ -50,18 +48,17 @@ export default function QuestionsPage() {
{publishedQa.map(row => (
-

{row.humanCorrectedQuestion ?? row.question}

+

{row.question}

- {row.humanCorrectedAnswer ?? row.response} + {row.answer}
-
- {row.toolName} - {row.publishedAt && ( - {format(toDate(row.publishedAt), "MMM d, yyyy")} - )} -
+ {row.publishedAt && ( +
+ {format(toDate(row.publishedAt), "MMM d, yyyy")} +
+ )}
))}
diff --git a/apps/internal-tool/src/components/CallLogDetail.tsx b/apps/internal-tool/src/components/CallLogDetail.tsx index 56b00cee69..0edb1fe672 100644 --- a/apps/internal-tool/src/components/CallLogDetail.tsx +++ b/apps/internal-tool/src/components/CallLogDetail.tsx @@ -1,9 +1,11 @@ +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; +import { runAsynchronouslyWithAlert } from "@stackframe/stack-shared/dist/utils/promises"; import { clsx } from "clsx"; import { format, formatDistanceToNow } from "date-fns"; import { useState, useEffect } from "react"; import Markdown from "react-markdown"; import remarkGfm from "remark-gfm"; -import type { McpCallLogRow } from "../types"; +import type { McpCallLogRow, QaEntriesRow } from "../types"; import { toDate } from "../utils"; import { ConversationReplay } from "./ConversationReplay"; import { markdownComponents } from "./markdown-components"; @@ -31,15 +33,51 @@ function CopyButton({ text }: { text: string }) { // ─── Main Component ──────────────────────────────────── -export function CallLogDetail({ row, allRows, onClose, onSaveCorrection, onMarkReviewed }: { +export function CallLogDetail({ row, allRows, qaEntries, onClose, onSaveCorrection, onMarkReviewed, onUnmarkReviewed }: { row: McpCallLogRow; allRows: McpCallLogRow[]; + qaEntries: QaEntriesRow[]; onClose: () => void; onSaveCorrection?: (correlationId: string, correctedQuestion: string, correctedAnswer: string, publish: boolean) => Promise | void; onMarkReviewed?: (correlationId: string) => Promise | void; + onUnmarkReviewed?: (correlationId: string) => Promise | void; }) { + const linkedQa = qaEntries.find(q => q.sourceMcpCorrelationId === row.correlationId); const [showReplay, setShowReplay] = useState(false); - const isReviewed = row.humanReviewedAt != null; + // Optimistic override while the mark/unmark roundtrip is in flight. Cleared + // once the real subscription update catches up. + const [optimisticReviewed, setOptimisticReviewed] = useState(null); + useEffect(() => { + const actual = row.humanReviewedAt != null; + if (optimisticReviewed != null && optimisticReviewed === actual) { + setOptimisticReviewed(null); + } + }, [row.humanReviewedAt, optimisticReviewed]); + const isReviewed = optimisticReviewed ?? (row.humanReviewedAt != null); + + const handleMark = () => { + const previous = optimisticReviewed; + setOptimisticReviewed(true); + runAsynchronouslyWithAlert( + Promise.resolve(onMarkReviewed?.(row.correlationId)).catch(err => { + // Revert the optimistic override so the UI reflects the database's real state. + setOptimisticReviewed(previous); + captureError("call-log-mark-reviewed", err); + throw err; + }) + ); + }; + const handleUnmark = () => { + const previous = optimisticReviewed; + setOptimisticReviewed(false); + runAsynchronouslyWithAlert( + Promise.resolve(onUnmarkReviewed?.(row.correlationId)).catch(err => { + setOptimisticReviewed(previous); + captureError("call-log-unmark-reviewed", err); + throw err; + }) + ); + }; return (
@@ -54,21 +92,33 @@ export function CallLogDetail({ row, allRows, onClose, onSaveCorrection, onMarkR {isReviewed && ( - ✓ Reviewed{row.humanReviewedBy ? ` by ${row.humanReviewedBy}` : ""} + ✓ Reviewed + {row.humanReviewedBy ? ` by ${row.humanReviewedBy}` : ""} + {row.humanReviewedAt + ? ` · ${formatDistanceToNow(toDate(row.humanReviewedAt), { addSuffix: true })}` + : " · just now"} )}
{!isReviewed && onMarkReviewed && ( )} + {isReviewed && onUnmarkReviewed && ( + + )}
); } @@ -374,20 +424,26 @@ async function fetchDeepWikiAnswer(questionText: string): Promise { .join("\n\n") ?? "(no response)"; } -function HumanCorrectionCard({ row, onSave }: { +function HumanCorrectionCard({ row, qa, onSave }: { row: McpCallLogRow; + qa: QaEntriesRow | undefined; onSave?: (correlationId: string, correctedQuestion: string, correctedAnswer: string, publish: boolean) => Promise | void; }) { - const [question, setQuestion] = useState(row.humanCorrectedQuestion ?? ""); - const [answer, setAnswer] = useState(row.humanCorrectedAnswer ?? ""); + const persistedQuestion = qa?.question ?? ""; + const persistedAnswer = qa?.answer ?? ""; + const isPublished = qa?.published === true; + const hasDraft = qa != null; + + const [question, setQuestion] = useState(persistedQuestion); + const [answer, setAnswer] = useState(persistedAnswer); const [lastAction, setLastAction] = useState<"published" | "saved" | "deepwiki-error" | "error" | null>(null); const [deepWikiLoading, setDeepWikiLoading] = useState(false); const [isSaving, setIsSaving] = useState(false); useEffect(() => { - setQuestion(row.humanCorrectedQuestion ?? ""); - setAnswer(row.humanCorrectedAnswer ?? ""); - }, [row.humanCorrectedQuestion, row.humanCorrectedAnswer, row.correlationId]); + setQuestion(persistedQuestion); + setAnswer(persistedAnswer); + }, [persistedQuestion, persistedAnswer, row.correlationId]); const handleSave = async (publish: boolean) => { if (isSaving) return; @@ -405,12 +461,12 @@ function HumanCorrectionCard({ row, onSave }: { }; const hasUnsavedChanges = - question !== (row.humanCorrectedQuestion ?? "") || - answer !== (row.humanCorrectedAnswer ?? ""); + question !== persistedQuestion || + answer !== persistedAnswer; - const cardStyle = row.publishedToQa + const cardStyle = isPublished ? "bg-green-50/50 border-green-200" - : row.humanCorrectedAnswer + : hasDraft ? "bg-amber-50/50 border-amber-200" : "bg-white border-gray-200"; @@ -420,24 +476,24 @@ function HumanCorrectionCard({ row, onSave }: {

Human Correction

- {row.publishedToQa ? ( + {isPublished ? ( ✓ Published - ) : row.humanCorrectedAnswer ? ( + ) : hasDraft ? ( Draft ) : null}
- {row.publishedAt && ( - {format(toDate(row.publishedAt), "MMM d, yyyy")} + {qa?.lastPublishedAt && ( + {format(toDate(qa.lastPublishedAt), "MMM d, yyyy")} )} - {row.humanReviewedBy && ( - by {row.humanReviewedBy} + {qa?.lastEditedBy && ( + by {qa.lastEditedBy} )} - {row.publishedToQa && ( + {isPublished && (
diff --git a/apps/internal-tool/src/components/CallLogList.tsx b/apps/internal-tool/src/components/CallLogList.tsx index 553b228ab1..834436dc6c 100644 --- a/apps/internal-tool/src/components/CallLogList.tsx +++ b/apps/internal-tool/src/components/CallLogList.tsx @@ -1,4 +1,4 @@ -import { useState, useMemo } from "react"; +import { useState, useMemo, useEffect } from "react"; import { formatDistanceToNow, format } from "date-fns"; import type { McpCallLogRow } from "../types"; import { toDate } from "../utils"; @@ -8,10 +8,12 @@ function truncate(str: string, max: number): string { return str.length > max ? str.slice(0, max) + "..." : str; } -type SortField = "time" | "tool" | "steps" | "duration" | "qa" | "status"; +type SortField = "time" | "tool" | "steps" | "duration" | "qa" | "reviewed" | "status"; type SortDir = "asc" | "desc"; type StatusFilter = "all" | "ok" | "error"; type QaFilter = "all" | "pending" | "pass" | "warn" | "fail" | "error" | "needs-review" | "human-reviewed" | "not-reviewed"; +const PAGE_SIZES = [25, 50, 100, 500] as const; +type PageSize = typeof PAGE_SIZES[number]; function getSortValue(row: McpCallLogRow, field: SortField): number | string { switch (field) { @@ -20,6 +22,7 @@ function getSortValue(row: McpCallLogRow, field: SortField): number | string { case "steps": { return row.stepCount; } case "duration": { return Number(row.durationMs); } case "qa": { return row.qaOverallScore ?? -1; } + case "reviewed": { return row.humanReviewedAt ? Number(toDate(row.humanReviewedAt).getTime()) : 0; } case "status": { return row.errorMessage ? 1 : 0; } } } @@ -56,6 +59,8 @@ export function CallLogList({ }, [rows]); const [toolFilter, setToolFilter] = useState("all"); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(50); const filteredAndSorted = useMemo(() => { let result = rows; @@ -115,6 +120,14 @@ export function CallLogList({ return result; }, [rows, textFilter, toolFilter, statusFilter, qaFilter, sortField, sortDir]); + const pageCount = Math.max(1, Math.ceil(filteredAndSorted.length / pageSize)); + const currentPage = Math.min(page, pageCount - 1); + const pageRows = filteredAndSorted.slice(currentPage * pageSize, (currentPage + 1) * pageSize); + + useEffect(() => { + setPage(0); + }, [textFilter, toolFilter, statusFilter, qaFilter, sortField, sortDir, pageSize]); + if (connectionState === "connecting") { return
Connecting to SpacetimeDB...
; } @@ -218,10 +231,6 @@ export function CallLogList({ ) : ( <>

No MCP calls logged yet

-

- Make sure STACK_MCP_LOG_TOKEN is set - in the backend and the SpacetimeDB module is published. -

)}
@@ -237,11 +246,12 @@ export function CallLogList({ Steps Duration QA + Human Reviewed Status - {filteredAndSorted.map((row) => ( + {pageRows.map((row) => ( onSelect(row)} @@ -287,11 +297,20 @@ export function CallLogList({ ) : ( -- )} - {row.humanReviewedAt && ( - - )} + + {row.humanReviewedAt ? ( + + ✓ {formatDistanceToNow(toDate(row.humanReviewedAt), { addSuffix: true })} + + ) : ( + -- + )} + {row.errorMessage ? ( @@ -307,6 +326,45 @@ export function CallLogList({ ))} +
+
+ Page size + {PAGE_SIZES.map(s => ( + + ))} +
+
+ + {filteredAndSorted.length === 0 + ? "No results" + : `${currentPage * pageSize + 1}–${Math.min((currentPage + 1) * pageSize, filteredAndSorted.length)} of ${filteredAndSorted.length}`} + + + {currentPage + 1} / {pageCount} + +
+
)}
diff --git a/apps/internal-tool/src/components/ConversationReplay.tsx b/apps/internal-tool/src/components/ConversationReplay.tsx index 02516dbe39..fe34388442 100644 --- a/apps/internal-tool/src/components/ConversationReplay.tsx +++ b/apps/internal-tool/src/components/ConversationReplay.tsx @@ -40,7 +40,7 @@ function CopyButton({ text }: { text: string }) { ); } -function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; args: unknown; result: unknown }; accent?: "purple" | "indigo" }) { +export function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; args: unknown; result: unknown }; accent?: "purple" | "indigo" }) { const [expanded, setExpanded] = useState(false); const colors = accent === "indigo" ? { dot: "text-indigo-500", name: "text-indigo-700", bg: "bg-indigo-50", ring: "ring-indigo-200", hover: "hover:bg-indigo-100" } @@ -82,7 +82,7 @@ function ToolCallCard({ call, accent = "purple" }: { call: { toolName: string; a ); } -function UserBubble({ text }: { text: string }) { +export function UserBubble({ text }: { text: string }) { return (
@@ -95,7 +95,7 @@ function UserBubble({ text }: { text: string }) { ); } -function AssistantBubble({ content, toolCalls }: { content: string; toolCalls: ToolCall[] }) { +export function AssistantBubble({ content, toolCalls }: { content: string; toolCalls: ToolCall[] }) { return (
@@ -565,8 +565,8 @@ export function ConversationReplay({ row, allRows, onClose }: { row: McpCallLogR {/* Footer */}
- {totalSteps} step{totalSteps !== 1 ? "s" : ""} · {totalDuration.toLocaleString()}ms - {isMultiCall && ` · ${conversationRows.length} calls`} + {totalSteps} step{totalSteps !== 1 ? "s" : ""} {"\u00B7"} {totalDuration.toLocaleString()}ms + {isMultiCall && ` \u00B7 ${conversationRows.length} calls`} {currentRow.modelId}
diff --git a/apps/internal-tool/src/components/KnowledgeBase.tsx b/apps/internal-tool/src/components/KnowledgeBase.tsx index baeffa4866..5d682da41f 100644 --- a/apps/internal-tool/src/components/KnowledgeBase.tsx +++ b/apps/internal-tool/src/components/KnowledgeBase.tsx @@ -1,46 +1,47 @@ +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { useState, useMemo } from "react"; import { format } from "date-fns"; import { clsx } from "clsx"; -import type { McpCallLogRow } from "../types"; +import type { QaEntriesRow } from "../types"; import { toDate } from "../utils"; type KbFilter = "all" | "published" | "draft"; export function KnowledgeBase({ rows, onSave, onDelete }: { - rows: McpCallLogRow[]; - onSave: (correlationId: string, question: string, answer: string, publish: boolean) => void; - onDelete: (correlationId: string) => void; + rows: QaEntriesRow[]; + onSave: (qaId: bigint, question: string, answer: string, publish: boolean) => Promise | void; + onDelete: (qaId: bigint) => Promise | void; }) { const [filter, setFilter] = useState("all"); const [search, setSearch] = useState(""); - const [editingId, setEditingId] = useState(null); + const [editingId, setEditingId] = useState(null); const kbRows = useMemo(() => { - let result = rows.filter(r => (r.humanCorrectedAnswer != null && r.humanCorrectedAnswer !== "") || r.publishedToQa); + let result = rows; if (filter === "published") { - result = result.filter(r => r.publishedToQa); + result = result.filter(r => r.published); } else if (filter === "draft") { - result = result.filter(r => !r.publishedToQa); + result = result.filter(r => !r.published); } if (search) { const lower = search.toLowerCase(); result = result.filter(r => - (r.humanCorrectedQuestion ?? r.question).toLowerCase().includes(lower) || - (r.humanCorrectedAnswer ?? r.response).toLowerCase().includes(lower) + r.question.toLowerCase().includes(lower) || + r.answer.toLowerCase().includes(lower) ); } - return result.sort((a, b) => { - const aTime = a.publishedAt ? Number(toDate(a.publishedAt)) : Number(toDate(a.createdAt)); - const bTime = b.publishedAt ? Number(toDate(b.publishedAt)) : Number(toDate(b.createdAt)); + return result.slice().sort((a, b) => { + const aTime = a.lastPublishedAt ? Number(toDate(a.lastPublishedAt)) : Number(toDate(a.createdAt)); + const bTime = b.lastPublishedAt ? Number(toDate(b.lastPublishedAt)) : Number(toDate(b.createdAt)); return bTime - aTime; }); }, [rows, filter, search]); - const publishedCount = rows.filter(r => r.publishedToQa).length; - const draftCount = rows.filter(r => r.humanCorrectedAnswer != null && r.humanCorrectedAnswer !== "" && !r.publishedToQa).length; + const publishedCount = rows.filter(r => r.published).length; + const draftCount = rows.filter(r => !r.published).length; return (
@@ -96,14 +97,18 @@ export function KnowledgeBase({ rows, onSave, onDelete }: { setEditingId(row.correlationId)} + isEditing={editingId === row.id} + onStartEdit={() => setEditingId(row.id)} onCancelEdit={() => setEditingId(null)} onSave={(question, answer, publish) => { - onSave(row.correlationId, question, answer, publish); + Promise.resolve(onSave(row.id, question, answer, publish)) + .catch(err => captureError("knowledge-base-save", err)); setEditingId(null); }} - onDelete={() => onDelete(row.correlationId)} + onDelete={() => { + Promise.resolve(onDelete(row.id)) + .catch(err => captureError("knowledge-base-delete", err)); + }} /> ))}
@@ -112,22 +117,56 @@ export function KnowledgeBase({ rows, onSave, onDelete }: { ); } +type PendingAction = "edit" | "publish" | "unpublish" | "delete" | null; + +function ConfirmDialog({ title, message, confirmLabel, confirmClassName, onConfirm, onCancel }: { + title: string; + message: string; + confirmLabel: string; + confirmClassName?: string; + onConfirm: () => void; + onCancel: () => void; +}) { + return ( +
+
e.stopPropagation()}> +

{title}

+

{message}

+
+ + +
+
+
+ ); +} + function KbCard({ row, isEditing, onStartEdit, onCancelEdit, onSave, onDelete }: { - row: McpCallLogRow; + row: QaEntriesRow; isEditing: boolean; onStartEdit: () => void; onCancelEdit: () => void; onSave: (question: string, answer: string, publish: boolean) => void; onDelete: () => void; }) { - const question = row.humanCorrectedQuestion ?? row.question; - const answer = row.humanCorrectedAnswer ?? row.response; + const isManual = row.sourceMcpCorrelationId == null; - const [editQuestion, setEditQuestion] = useState(question); - const [editAnswer, setEditAnswer] = useState(answer); + const [editQuestion, setEditQuestion] = useState(row.question); + const [editAnswer, setEditAnswer] = useState(row.answer); + const [pending, setPending] = useState(null); - const cardBorder = row.publishedToQa ? "border-green-200" : "border-amber-200"; - const cardBg = row.publishedToQa ? "bg-green-50/30" : "bg-amber-50/30"; + const cardBorder = row.published ? "border-green-200" : "border-amber-200"; + const cardBg = row.published ? "bg-green-50/30" : "bg-amber-50/30"; if (isEditing) { return ( @@ -166,7 +205,7 @@ function KbCard({ row, isEditing, onStartEdit, onCancelEdit, onSave, onDelete }: onClick={() => onSave(editQuestion, editAnswer, true)} className="px-3 py-1.5 text-xs font-medium text-white bg-blue-600 rounded-md hover:bg-blue-700" > - {row.publishedToQa ? "Update & Publish" : "Save & Publish"} + {row.published ? "Update & Publish" : "Save & Publish"}
@@ -178,7 +217,7 @@ function KbCard({ row, isEditing, onStartEdit, onCancelEdit, onSave, onDelete }: {/* Header */}
- {row.publishedToQa ? ( + {row.published ? ( ✓ Published @@ -187,42 +226,42 @@ function KbCard({ row, isEditing, onStartEdit, onCancelEdit, onSave, onDelete }: Draft )} - {row.toolName === "manual" && ( + {isManual && ( manual )} - {row.publishedAt && ( + {row.lastPublishedAt && ( - {format(toDate(row.publishedAt), "MMM d, yyyy")} + {format(toDate(row.lastPublishedAt), "MMM d, yyyy")} )} - {row.humanReviewedBy && ( - by {row.humanReviewedBy} + {row.lastEditedBy && ( + by {row.lastEditedBy} )}
- {row.publishedToQa ? ( + {row.published ? ( ) : ( )}
); } diff --git a/apps/internal-tool/src/components/Usage.tsx b/apps/internal-tool/src/components/Usage.tsx new file mode 100644 index 0000000000..ae7e7cea93 --- /dev/null +++ b/apps/internal-tool/src/components/Usage.tsx @@ -0,0 +1,837 @@ +import { useEffect, useMemo, useState } from "react"; +import { clsx } from "clsx"; +import type { AiQueryLogRow } from "../types"; +import { toDate } from "../utils"; + +type TimeRange = "24h" | "7d" | "30d" | "all"; +type AuthFilter = "all" | "authed" | "anon"; +type ModeFilter = "all" | "stream" | "generate"; +type StatusFilter = "all" | "ok" | "error"; +type SortKey = "createdAt" | "systemPromptId" | "modelId" | "mode" | "inputTokens" | "outputTokens" | "cachedInputTokens" | "cacheCreationTokens" | "cacheSavingsUsd" | "costUsd" | "durationMs" | "status"; +type SortDir = "asc" | "desc"; +const PAGE_SIZES = [25, 50, 100, 500] as const; +type PageSize = typeof PAGE_SIZES[number]; + +type Props = { + rows: AiQueryLogRow[], + connectionState: "connecting" | "connected" | "error", + onSelect: (row: AiQueryLogRow) => void, + selectedId?: bigint, +}; + +const ALL_SYSTEM_PROMPTS = [ + "command-center-ask-ai", + "docs-ask-ai", + "wysiwyg-edit", + "email-wysiwyg-editor", + "email-assistant-template", + "email-assistant-theme", + "email-assistant-draft", + "create-dashboard", + "run-query", + "rewrite-template-source", +]; + +export function Usage({ rows, connectionState, onSelect, selectedId }: Props) { + const [timeRange, setTimeRange] = useState("7d"); + const [systemPromptFilter, setSystemPromptFilter] = useState>(new Set()); + const [modelFilter, setModelFilter] = useState>(new Set()); + const [modeFilter, setModeFilter] = useState("all"); + const [authFilter, setAuthFilter] = useState("all"); + const [statusFilter, setStatusFilter] = useState("all"); + const [search, setSearch] = useState(""); + const [sortKey, setSortKey] = useState("createdAt"); + const [sortDir, setSortDir] = useState("desc"); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(50); + + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + const id = setInterval(() => setNow(Date.now()), 60_000); + return () => clearInterval(id); + }, []); + const rangeStart = useMemo(() => { + switch (timeRange) { + case "24h": { + return now - 24 * 60 * 60 * 1000; + } + case "7d": { + return now - 7 * 24 * 60 * 60 * 1000; + } + case "30d": { + return now - 30 * 24 * 60 * 60 * 1000; + } + case "all": { + return 0; + } + } + }, [timeRange, now]); + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + return rows.filter(r => { + const ts = toDate(r.createdAt).getTime(); + if (ts < rangeStart) return false; + if (systemPromptFilter.size > 0 && !systemPromptFilter.has(r.systemPromptId)) return false; + if (modelFilter.size > 0 && !modelFilter.has(r.modelId)) return false; + if (modeFilter !== "all" && r.mode !== modeFilter) return false; + if (authFilter === "authed" && !r.isAuthenticated) return false; + if (authFilter === "anon" && r.isAuthenticated) return false; + const isError = r.errorMessage != null && r.errorMessage !== ""; + if (statusFilter === "ok" && isError) return false; + if (statusFilter === "error" && !isError) return false; + if (q) { + const hay = `${r.finalText} ${r.messagesJson}`.toLowerCase(); + if (!hay.includes(q)) return false; + } + return true; + }); + }, [rows, rangeStart, systemPromptFilter, modelFilter, modeFilter, authFilter, statusFilter, search]); + + const sorted = useMemo(() => { + const copy = [...filtered]; + const mult = sortDir === "asc" ? 1 : -1; + copy.sort((a, b) => { + let av: number | string = 0; + let bv: number | string = 0; + switch (sortKey) { + case "createdAt": { + av = toDate(a.createdAt).getTime(); + bv = toDate(b.createdAt).getTime(); + break; + } + case "systemPromptId": { + av = a.systemPromptId; + bv = b.systemPromptId; + break; + } + case "modelId": { + av = a.modelId; + bv = b.modelId; + break; + } + case "mode": { + av = a.mode; + bv = b.mode; + break; + } + case "inputTokens": { + av = a.inputTokens ?? -1; + bv = b.inputTokens ?? -1; + break; + } + case "outputTokens": { + av = a.outputTokens ?? -1; + bv = b.outputTokens ?? -1; + break; + } + case "cachedInputTokens": { + av = a.cachedInputTokens ?? -1; + bv = b.cachedInputTokens ?? -1; + break; + } + case "cacheCreationTokens": { + av = a.cacheCreationTokens ?? -1; + bv = b.cacheCreationTokens ?? -1; + break; + } + case "cacheSavingsUsd": { + av = a.cacheDiscountUsd ?? Number.NEGATIVE_INFINITY; + bv = b.cacheDiscountUsd ?? Number.NEGATIVE_INFINITY; + break; + } + case "costUsd": { + av = a.costUsd ?? -1; + bv = b.costUsd ?? -1; + break; + } + case "durationMs": { + av = Number(a.durationMs); + bv = Number(b.durationMs); + break; + } + case "status": { + av = (a.errorMessage != null && a.errorMessage !== "") ? 1 : 0; + bv = (b.errorMessage != null && b.errorMessage !== "") ? 1 : 0; + break; + } + } + if (av < bv) return -1 * mult; + if (av > bv) return 1 * mult; + return 0; + }); + return copy; + }, [filtered, sortKey, sortDir]); + + const pageCount = Math.max(1, Math.ceil(sorted.length / pageSize)); + const currentPage = Math.min(page, pageCount - 1); + const pageRows = sorted.slice(currentPage * pageSize, (currentPage + 1) * pageSize); + + function toggleSort(key: SortKey) { + if (sortKey === key) { + setSortDir(sortDir === "asc" ? "desc" : "asc"); + } else { + setSortKey(key); + setSortDir(key === "createdAt" ? "desc" : "asc"); + } + setPage(0); + } + + useEffect(() => { + setPage(0); + }, [timeRange, systemPromptFilter, modelFilter, modeFilter, authFilter, statusFilter, search, pageSize]); + + const stats = useMemo(() => { + const totalCalls = filtered.length; + const errorCalls = filtered.filter(r => r.errorMessage != null && r.errorMessage !== "").length; + const inputTokens = filtered.reduce((a, r) => a + (r.inputTokens ?? 0), 0); + const outputTokens = filtered.reduce((a, r) => a + (r.outputTokens ?? 0), 0); + const cachedInputTokens = filtered.reduce((a, r) => a + (r.cachedInputTokens ?? 0), 0); + const cacheCreationTokens = filtered.reduce((a, r) => a + (r.cacheCreationTokens ?? 0), 0); + const cacheSavingsUsd = filtered.reduce((a, r) => a + (r.cacheDiscountUsd ?? 0), 0); + const totalCost = filtered.reduce((a, r) => a + (r.costUsd ?? 0), 0); + const durations = filtered.map(r => Number(r.durationMs)).filter(d => d > 0).sort((a, b) => a - b); + const avgDuration = durations.length > 0 ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0; + const p95Duration = durations.length > 0 ? durations[Math.min(Math.floor(durations.length * 0.95), durations.length - 1)] : 0; + + let seriesStart: number; + let seriesEnd: number; + if (timeRange === "all" && filtered.length > 0) { + seriesStart = Infinity; + seriesEnd = -Infinity; + for (const r of filtered) { + const ts = toDate(r.createdAt).getTime(); + if (ts < seriesStart) seriesStart = ts; + if (ts > seriesEnd) seriesEnd = ts; + } + } else { + seriesStart = rangeStart; + seriesEnd = now; + } + const spanMs = Math.max(0, seriesEnd - seriesStart); + const bucketMs = spanMs <= 24 * 60 * 60 * 1000 ? 60 * 60 * 1000 : 24 * 60 * 60 * 1000; + const bucketLabelFmt: Intl.DateTimeFormatOptions = bucketMs === 60 * 60 * 1000 + ? { hour: "numeric" } + : { month: "short", day: "numeric" }; + const bucketCount = Math.min(48, Math.max(1, Math.ceil(spanMs / bucketMs))); + const bucketStart = seriesEnd - bucketCount * bucketMs; + const timeBuckets: Array<{ label: string, start: number, calls: number, inputTokens: number, outputTokens: number, cachedInputTokens: number }> = []; + for (let i = 0; i < bucketCount; i++) { + const start = bucketStart + i * bucketMs; + timeBuckets.push({ + label: new Date(start).toLocaleString("en-US", bucketLabelFmt), + start, + calls: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + }); + } + for (const r of filtered) { + const ts = toDate(r.createdAt).getTime(); + const idx = Math.floor((ts - bucketStart) / bucketMs); + if (idx >= 0 && idx < bucketCount) { + timeBuckets[idx].calls++; + timeBuckets[idx].inputTokens += r.inputTokens ?? 0; + timeBuckets[idx].outputTokens += r.outputTokens ?? 0; + timeBuckets[idx].cachedInputTokens += r.cachedInputTokens ?? 0; + } + } + const maxCalls = Math.max(...timeBuckets.map(b => b.calls), 1); + const maxTokenTotal = Math.max(...timeBuckets.map(b => b.inputTokens + b.outputTokens), 1); + const maxInputTokens = Math.max(...timeBuckets.map(b => b.inputTokens), 1); + + // Distributions + const sysPromptCounts = new Map(); + const modelCounts = new Map(); + const toolCounts = new Map(); + for (const r of filtered) { + sysPromptCounts.set(r.systemPromptId, (sysPromptCounts.get(r.systemPromptId) ?? 0) + 1); + modelCounts.set(r.modelId, (modelCounts.get(r.modelId) ?? 0) + 1); + try { + const tools = JSON.parse(r.requestedToolsJson) as string[]; + for (const t of tools) toolCounts.set(t, (toolCounts.get(t) ?? 0) + 1); + } catch { /* skip */ } + } + const sysPromptDist = Array.from(sysPromptCounts.entries()).sort((a, b) => b[1] - a[1]); + const modelDist = Array.from(modelCounts.entries()).sort((a, b) => b[1] - a[1]); + const toolDist = Array.from(toolCounts.entries()).sort((a, b) => b[1] - a[1]); + + // Cache Hit % per systemPromptId + const cacheBySystemPrompt = new Map(); + for (const r of filtered) { + const existing = cacheBySystemPrompt.get(r.systemPromptId) ?? { input: 0, cached: 0, calls: 0 }; + existing.input += r.inputTokens ?? 0; + existing.cached += r.cachedInputTokens ?? 0; + existing.calls += 1; + cacheBySystemPrompt.set(r.systemPromptId, existing); + } + const cacheHitBySystemPrompt = Array.from(cacheBySystemPrompt.entries()) + .map(([id, v]) => ({ + id, + calls: v.calls, + hitPct: v.input > 0 ? Math.round((v.cached / v.input) * 100) : 0, + cached: v.cached, + input: v.input, + })) + .sort((a, b) => b.input - a.input); + + // Latency histogram + const latencyBuckets = [ + { label: "<500ms", max: 500, count: 0 }, + { label: "500ms–2s", max: 2000, count: 0 }, + { label: "2–10s", max: 10000, count: 0 }, + { label: "10–30s", max: 30000, count: 0 }, + { label: ">30s", max: Infinity, count: 0 }, + ]; + for (const d of durations) { + const b = latencyBuckets.find(b => d < b.max); + if (b) b.count++; + } + const maxLatencyBucket = Math.max(...latencyBuckets.map(b => b.count), 1); + + return { + totalCalls, errorCalls, inputTokens, outputTokens, cachedInputTokens, cacheCreationTokens, cacheSavingsUsd, totalCost, + avgDuration, p95Duration, + timeBuckets, maxCalls, maxTokenTotal, maxInputTokens, + sysPromptDist, modelDist, toolDist, + cacheHitBySystemPrompt, + latencyBuckets, maxLatencyBucket, + }; + }, [filtered, rangeStart, now]); + + const allSystemPrompts = useMemo(() => { + const seen = new Set(ALL_SYSTEM_PROMPTS); + for (const r of rows) seen.add(r.systemPromptId); + return Array.from(seen).sort(); + }, [rows]); + + const allModels = useMemo(() => { + const seen = new Set(); + for (const r of rows) seen.add(r.modelId); + return Array.from(seen).sort(); + }, [rows]); + + function toggle(set: Set, val: string, setter: (s: Set) => void) { + const next = new Set(set); + if (next.has(val)) next.delete(val); + else next.add(val); + setter(next); + } + + return ( +
+ {/* Filter bar */} +
+
+ Range + {(["24h", "7d", "30d", "all"] as TimeRange[]).map(r => ( + + ))} + + Mode + {(["all", "stream", "generate"] as ModeFilter[]).map(m => ( + + ))} + + Auth + {(["all", "authed", "anon"] as AuthFilter[]).map(a => ( + + ))} + + Status + {(["all", "ok", "error"] as StatusFilter[]).map(s => ( + + ))} + + setSearch(e.target.value)} + placeholder="Search messages / response" + className="px-2 py-1 text-xs border border-gray-200 rounded w-64" + /> + + {connectionState === "connected" ? `${filtered.length} / ${rows.length} calls` : connectionState} + +
+
+ System prompt + {allSystemPrompts.map(sp => ( + + ))} +
+ {allModels.length > 0 && ( +
+ Model + {allModels.map(m => ( + + ))} +
+ )} +
+ + {/* Metric cards */} +
+ + 0 ? "text-red-600" : undefined} + tooltip="Requests that failed. Counted as rows where errorMessage is non-empty (upstream provider error, timeout, or client abort)." + /> + + + 0 ? `${Math.round((stats.cachedInputTokens / stats.inputTokens) * 100)}%` : "—"} + valueClass={stats.inputTokens > 0 && stats.cachedInputTokens / stats.inputTokens > 0.5 ? "text-green-600" : undefined} + tooltip="Share of input tokens served from cache vs. processed fresh. Computed as sum(cachedInputTokens) / sum(inputTokens). Higher = caching is doing its job." + /> + + = 0 ? "+" : "−"}${formatUsd(Math.abs(stats.cacheSavingsUsd))}`} + valueClass={stats.cacheSavingsUsd >= 0 ? "text-green-600" : "text-red-600"} + tooltip="Sum of cache_discount values across filtered requests. Positive (green) means caching net-saved money; negative (red) means cold-start writes outweighed reads. Filter by systemPromptId to judge whether caching is worth keeping on a specific flow." + /> + + +
+ + {/* Time-series charts */} +
+ +
+ {stats.timeBuckets.map((b, i) => ( +
+
+
+
+
+ ))} +
+
+ {stats.timeBuckets[0]?.label} + {stats.timeBuckets[stats.timeBuckets.length - 1]?.label} +
+ + + +
+ {stats.timeBuckets.map((b, i) => { + const total = b.inputTokens + b.outputTokens; + const outPct = total > 0 ? (b.outputTokens / total) * 100 : 0; + return ( +
+
+
+
+ ); + })} +
+
+ input + output +
+ + + +
+ {stats.timeBuckets.map((b, i) => { + const cachedPct = b.inputTokens > 0 ? (b.cachedInputTokens / b.inputTokens) * 100 : 0; + return ( +
+
+
+
+ ); + })} +
+
+ fresh + cached +
+ + + + {stats.cacheHitBySystemPrompt.length === 0 ? ( +

No data

+ ) : ( +
+ {stats.cacheHitBySystemPrompt.map(entry => ( +
+ {entry.id} +
+
= 50 ? "bg-green-500" : entry.hitPct >= 20 ? "bg-yellow-400" : "bg-red-400" + )} + style={{ width: `${entry.hitPct}%` }} + /> +
+ {entry.hitPct}% + {entry.calls} calls +
+ ))} +
+ )} + + + + + + + + + + + + + + + +
+ {stats.latencyBuckets.map(b => ( +
+ {b.label} +
+
+
+ {b.count} +
+ ))} +
+ +
+ + {/* Call list */} + +
+ + + + toggleSort("createdAt")} tooltip="When the request was logged.">Time + toggleSort("systemPromptId")} tooltip="Which app flow triggered the AI call (e.g. create-dashboard, docs-ask-ai).">System Prompt + toggleSort("modelId")} tooltip="Which LLM processed the request.">Model + toggleSort("mode")} tooltip="stream — tokens streamed as generated (streamText). generate — single JSON response after completion (generateText).">Mode + toggleSort("inputTokens")} tooltip="Total prompt tokens sent to the model">In tok + toggleSort("outputTokens")} tooltip="Tokens the model generated in its response.">Out tok + toggleSort("cachedInputTokens")} tooltip="Prompt tokens served from Anthropic's prompt cache. Higher = caching is paying off.">Cached + toggleSort("cacheCreationTokens")} tooltip="Prompt tokens written to cache on this request. High on cold-start; should be near zero on warm hits.">Cache W + toggleSort("cacheSavingsUsd")} tooltip="Dollars saved by caching on this request, computed by OpenRouter.">Cache $ + toggleSort("costUsd")} tooltip="Total dollar cost of this request, billed by OpenRouter. Includes prompt, completion, cache reads, and cache writes.">Cost + toggleSort("durationMs")} tooltip="Total wall-clock time from request start to onFinish, in milliseconds. Measured via performance.now() in the backend handler.">Duration + toggleSort("status")} tooltip="ok = success. error = upstream provider returned an error, the request was aborted, or the AI SDK threw.">Status + + + + {pageRows.map(row => { + const isError = row.errorMessage != null && row.errorMessage !== ""; + return ( + onSelect(row)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onSelect(row); + } + }} + className={clsx( + "border-b border-gray-100 cursor-pointer hover:bg-blue-50 focus:outline-none focus:ring-2 focus:ring-blue-500", + selectedId === row.id && "bg-blue-50" + )} + > + + + + + + + + + + + + + + ); + })} + +
+ {toDate(row.createdAt).toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" })} + + + {row.systemPromptId} + + {row.conversationId != null && ( + MCP + )} + {!row.isAuthenticated && ( + anon + )} + {row.modelId}{row.mode}{row.inputTokens?.toLocaleString() ?? "—"}{row.outputTokens?.toLocaleString() ?? "—"} + {row.cachedInputTokens != null && row.cachedInputTokens > 0 ? ( + {row.cachedInputTokens.toLocaleString()} + ) : ( + + )} + + {row.cacheCreationTokens != null && row.cacheCreationTokens > 0 ? ( + {row.cacheCreationTokens.toLocaleString()} + ) : ( + + )} + + {(() => { + const savings = row.cacheDiscountUsd; + if (savings == null) return ; + const sign = savings >= 0 ? "+" : "−"; + const color = savings >= 0 ? "text-green-600" : "text-red-600"; + return {sign}{formatUsd(Math.abs(savings))}; + })()} + {row.costUsd != null ? formatUsd(row.costUsd) : "—"}{Number(row.durationMs).toLocaleString()}ms + {isError ? ( + error + ) : ( + ok + )} +
+
+
+ Page size + {PAGE_SIZES.map(s => ( + + ))} +
+
+ + {sorted.length === 0 + ? "No results" + : `${currentPage * pageSize + 1}–${Math.min((currentPage + 1) * pageSize, sorted.length)} of ${sorted.length}`} + + + {currentPage + 1} / {pageCount} + +
+
+
+
+
+ ); +} + +function SortHeader({ + children, + align, + active, + dir, + onClick, + tooltip, +}: { + children: React.ReactNode, + align: "left" | "right", + active: boolean, + dir: SortDir, + onClick: () => void, + tooltip?: string, +}) { + return ( + + + {tooltip != null && ( +
+ {tooltip} +
+ )} + + ); +} + +function formatUsd(value: number): string { + if (value === 0) return "$0"; + return `$${value.toFixed(4)}`; +} + +function MetricCard({ label, value, valueClass, tooltip }: { label: string, value: string, valueClass?: string, tooltip?: string }) { + return ( +
+

{label}

+

{value}

+ {tooltip != null && ( +
+ {tooltip} +
+ )} +
+ ); +} + +function Card({ title, children }: { title: string, children: React.ReactNode }) { + return ( +
+

{title}

+ {children} +
+ ); +} + +function DistributionBars({ items, color }: { items: Array<[string, number]>, color: string }) { + if (items.length === 0) { + return

No data

; + } + const max = Math.max(...items.map(i => i[1]), 1); + return ( +
+ {items.map(([label, count]) => ( +
+ {label} +
+
+
+ {count} +
+ ))} +
+ ); +} diff --git a/apps/internal-tool/src/components/UsageDetail.tsx b/apps/internal-tool/src/components/UsageDetail.tsx new file mode 100644 index 0000000000..b07617786c --- /dev/null +++ b/apps/internal-tool/src/components/UsageDetail.tsx @@ -0,0 +1,226 @@ +import { useMemo } from "react"; +import Markdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import type { AiQueryLogRow } from "../types"; +import { toDate } from "../utils"; +import { AssistantBubble, ToolCallCard, UserBubble } from "./ConversationReplay"; +import { markdownComponents } from "./markdown-components"; + +type MessageIn = { + role: "user" | "assistant" | "tool", + content: unknown, +}; + +type StepEntry = { + step: number, + text?: string, + toolCalls?: Array<{ toolName: string, toolCallId: string, args: unknown }>, + toolResults?: Array<{ toolName: string, toolCallId: string, result: unknown }>, +}; + +function messageContentToText(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map(part => { + if (typeof part === "string") return part; + if (part && typeof part === "object" && "text" in part) return String((part as { text: unknown }).text ?? ""); + return JSON.stringify(part); + }) + .join(""); + } + if (content == null) return ""; + return JSON.stringify(content); +} + +export function UsageDetail({ row, onClose }: { row: AiQueryLogRow, onClose: () => void }) { + const messages: MessageIn[] = useMemo(() => { + try { + return JSON.parse(row.messagesJson) as MessageIn[]; + } catch { + return []; + } + }, [row.messagesJson]); + + const steps: StepEntry[] = useMemo(() => { + try { + return JSON.parse(row.stepsJson) as StepEntry[]; + } catch { + return []; + } + }, [row.stepsJson]); + + const requestedTools: string[] = useMemo(() => { + try { + return JSON.parse(row.requestedToolsJson) as string[]; + } catch { + return []; + } + }, [row.requestedToolsJson]); + + const assistantBubbles = steps.map((s, i) => { + const toolCalls = (s.toolCalls ?? []).map((tc, idx) => { + const matched = s.toolResults?.find(r => r.toolCallId === tc.toolCallId) ?? s.toolResults?.[idx]; + return { + type: "tool-call", + toolName: tc.toolName, + toolCallId: tc.toolCallId, + args: tc.args, + result: matched?.result ?? null, + }; + }); + return { key: i, text: s.text ?? "", toolCalls }; + }); + + const isError = row.errorMessage != null && row.errorMessage !== ""; + + return ( +
+
+
+
+ + {row.systemPromptId} + + + {row.modelId} + + + {row.mode} + + {isError && ( + error + )} + {row.conversationId != null && ( + MCP + )} +
+

+ {toDate(row.createdAt).toLocaleString()} + {" · "}{Number(row.durationMs).toLocaleString()}ms + {" · "}in {row.inputTokens?.toLocaleString() ?? "?"} tok + {(() => { + const r = row.cachedInputTokens ?? 0; + const w = row.cacheCreationTokens ?? 0; + if (r === 0 && w === 0) return null; + const parts: string[] = []; + if (r > 0) parts.push(`r ${r.toLocaleString()}`); + if (w > 0) parts.push(`w ${w.toLocaleString()}`); + return <> (cache {parts.join(", ")}); + })()} + {" · "}out {row.outputTokens?.toLocaleString() ?? "?"} tok + {row.costUsd != null && <>{" · "}${row.costUsd.toFixed(4)}} + {(() => { + const savings = row.cacheDiscountUsd; + if (savings == null) return null; + const sign = savings >= 0 ? "+" : "−"; + const color = savings >= 0 ? "text-green-600" : "text-red-600"; + return <>{" · "}cache {sign}${Math.abs(savings).toFixed(4)}; + })()} +

+
+ +
+ +
+ {isError && ( +
+

Error

+
{row.errorMessage}
+
+ )} + + {/* Metadata panel */} +
+ + + {row.projectId && } + {row.userId && } + {row.conversationId && } + + 0 ? requestedTools.join(", ") : "—"} /> +
+ + {/* Conversation replay */} +
+

Input Messages

+ {messages.length === 0 && ( +

No input messages.

+ )} + {messages.map((m, i) => { + const text = messageContentToText(m.content); + if (m.role === "user") { + return ; + } + if (m.role === "assistant") { + return ; + } + return ( +
+
+ T +
+
+
{text}
+
+
+ ); + })} + +

Assistant Steps

+ {assistantBubbles.length === 0 && ( +

No assistant output recorded.

+ )} + {assistantBubbles.map(bubble => ( +
+ {bubble.toolCalls.length > 0 && ( +
+ {bubble.toolCalls.map((call, i) => ( + + ))} +
+ )} + {bubble.text && ( +
+
+ AI +
+
+ + {bubble.text} + +
+
+ )} +
+ ))} + + {row.finalText && assistantBubbles.length === 0 && ( + <> +

Final Response

+
+ + {row.finalText} + +
+ + )} +
+
+
+ ); +} + +function MetaRow({ label, value }: { label: string, value: string }) { + return ( +
+ {label} + {value} +
+ ); +} diff --git a/apps/internal-tool/src/hooks/useSpacetimeDB.ts b/apps/internal-tool/src/hooks/useSpacetimeDB.ts index 65efa1548b..a6b1a1740f 100644 --- a/apps/internal-tool/src/hooks/useSpacetimeDB.ts +++ b/apps/internal-tool/src/hooks/useSpacetimeDB.ts @@ -1,46 +1,63 @@ +import { captureError } from "@stackframe/stack-shared/dist/utils/errors"; import { useEffect, useState, useRef } from "react"; -import { DbConnection, type EventContext, type SubscriptionEventContext } from "../module_bindings"; -import type { McpCallLogRow } from "../types"; - -const IS_DEV = process.env.NODE_ENV === "development"; -const PLACEHOLDER = "REPLACE_ME"; -const rawHost = process.env.NEXT_PUBLIC_SPACETIMEDB_HOST; -const rawDbName = process.env.NEXT_PUBLIC_SPACETIMEDB_DB_NAME; -function resolveEnv(raw: string | undefined, devDefault: string, name: string): string { - if (raw && raw !== PLACEHOLDER) return raw; - if (IS_DEV) return devDefault; - throw new Error(`${name} is not configured. Set it in .env.local or hosting platform env.`); +import type { Identity } from "spacetimedb"; +import { envOrDevDefault } from "../lib/env"; +import { DbConnection, type ErrorContext, type EventContext, type SubscriptionEventContext } from "../module_bindings"; +import type { AiQueryLogRow, McpCallLogRow, PublishedQaRow, QaEntriesRow } from "../types"; + +export type EnsureEnrolled = (identity: Identity) => Promise; + +let cachedConfig: { host: string, dbName: string, tokenKey: string } | null = null; +function getConfig() { + if (cachedConfig) return cachedConfig; + const host = envOrDevDefault(process.env.NEXT_PUBLIC_SPACETIMEDB_HOST, "ws://localhost:8139", "NEXT_PUBLIC_SPACETIMEDB_HOST"); + if (process.env.NODE_ENV !== "development" && !host.startsWith("wss://")) { + throw new Error("NEXT_PUBLIC_SPACETIMEDB_HOST must use wss:// in production"); + } + const dbName = envOrDevDefault(process.env.NEXT_PUBLIC_SPACETIMEDB_DB_NAME, "stack-auth-llm", "NEXT_PUBLIC_SPACETIMEDB_DB_NAME"); + cachedConfig = { host, dbName, tokenKey: `spacetimedb_${host}/${dbName}/auth_token` }; + return cachedConfig; } -const HOST = resolveEnv(rawHost, "ws://localhost:8139", "NEXT_PUBLIC_SPACETIMEDB_HOST"); -const DB_NAME = resolveEnv(rawDbName, "stack-auth-llm", "NEXT_PUBLIC_SPACETIMEDB_DB_NAME"); -const TOKEN_KEY = `spacetimedb_${HOST}/${DB_NAME}/auth_token`; const MAX_RETRIES = 5; const RETRY_DELAY_MS = 2000; -type ConnectionState = "connecting" | "connected" | "disconnected" | "error"; +type ConnectionState = "connecting" | "connected" | "error"; -export function useMcpCallLogs() { - const [rows, setRows] = useState([]); +type TableBinding = { + tableName: string, + iter: (ctx: SubscriptionEventContext) => Iterable, + onInsert: (conn: DbConnection, cb: (row: Row) => void) => void, + onDelete: (conn: DbConnection, cb: (row: Row) => void) => void, + onUpdate?: (conn: DbConnection, cb: (row: Row) => void) => void, +}; + +function useTableSubscription( + binding: TableBinding, + ensureEnrolled?: EnsureEnrolled, +) { + const [rows, setRows] = useState([]); const [connectionState, setConnectionState] = useState("connecting"); const connRef = useRef(null); + const ensureEnrolledRef = useRef(ensureEnrolled); + useEffect(() => { + ensureEnrolledRef.current = ensureEnrolled; + }, [ensureEnrolled]); useEffect(() => { let cancelled = false; let retryCount = 0; let retryTimer: ReturnType | null = null; - - console.log("[SpacetimeDB] Connecting to", HOST, "db:", DB_NAME); + const query = `SELECT * FROM ${binding.tableName}`; function retry() { if (cancelled) return; retryCount++; if (retryCount > MAX_RETRIES) { - console.error("[SpacetimeDB] Max retries reached"); + captureError("spacetimedb-connect-max-retries", new Error(`Gave up connecting to ${binding.tableName} after ${MAX_RETRIES} retries`)); setConnectionState("error"); return; } - console.log(`[SpacetimeDB] Retrying in ${RETRY_DELAY_MS}ms (attempt ${retryCount}/${MAX_RETRIES})...`); retryTimer = setTimeout(() => { retryTimer = null; if (!cancelled) { @@ -50,32 +67,52 @@ export function useMcpCallLogs() { } function connect() { + const config = getConfig(); const conn = DbConnection.builder() - .withUri(HOST) - .withDatabaseName(DB_NAME) - .withToken(localStorage.getItem(TOKEN_KEY) || undefined) - .onConnect((connInstance: DbConnection, _identity: unknown, token: string) => { + .withUri(config.host) + .withDatabaseName(config.dbName) + .withToken(localStorage.getItem(config.tokenKey) || undefined) + .onConnect((connInstance: DbConnection, identity: Identity, token: string) => { if (cancelled) return; - console.log("[SpacetimeDB] Connected successfully"); retryCount = 0; - localStorage.setItem(TOKEN_KEY, token); + localStorage.setItem(config.tokenKey, token); connRef.current = connInstance; - connInstance.subscriptionBuilder() - .onApplied((ctx: SubscriptionEventContext) => { - if (cancelled) return; - const initialRows: McpCallLogRow[] = []; - for (const row of ctx.db.mcpCallLog.iter()) { - initialRows.push(row); - } - initialRows.sort((a, b) => Number(b.id - a.id)); - console.log("[SpacetimeDB] Loaded", initialRows.length, "rows"); - setRows(initialRows); - setConnectionState("connected"); - }) - .subscribe(`SELECT * FROM mcp_call_log`); - - connInstance.db.mcpCallLog.onInsert((_ctx: EventContext, row: McpCallLogRow) => { + const startSubscription = () => { + if (cancelled) return; + connInstance.subscriptionBuilder() + .onApplied((ctx: SubscriptionEventContext) => { + if (cancelled) return; + const initial: Row[] = []; + for (const row of binding.iter(ctx)) { + initial.push(row); + } + initial.sort((a, b) => Number(b.id - a.id)); + setRows(initial); + setConnectionState("connected"); + }) + .onError((ctx: ErrorContext) => { + if (cancelled) return; + captureError("spacetimedb-subscription", ctx); + setConnectionState("error"); + }) + .subscribe(query); + }; + + const enrollFn = ensureEnrolledRef.current; + if (enrollFn) { + enrollFn(identity).then( + () => startSubscription(), + (err) => { + captureError("spacetimedb-enroll", err); + setConnectionState("error"); + }, + ); + } else { + startSubscription(); + } + + binding.onInsert(connInstance, (row) => { if (cancelled) return; setRows(prev => { const existing = prev.findIndex(r => r.id === row.id); @@ -88,17 +125,28 @@ export function useMcpCallLogs() { }); }); - connInstance.db.mcpCallLog.onDelete((_ctx: EventContext, row: McpCallLogRow) => { + binding.onDelete(connInstance, (row) => { if (cancelled) return; setRows(prev => prev.filter(r => r.id !== row.id)); }); + + binding.onUpdate?.(connInstance, (row) => { + if (cancelled) return; + setRows(prev => { + const idx = prev.findIndex(r => r.id === row.id); + if (idx < 0) return [row, ...prev]; + const updated = [...prev]; + updated[idx] = row; + return updated; + }); + }); }) .onConnectError((_ctx: unknown, err: unknown) => { - console.error("[SpacetimeDB] Connection error:", err); - const storedToken = localStorage.getItem(TOKEN_KEY); - if (storedToken) { - console.log("[SpacetimeDB] Clearing stale token"); - localStorage.removeItem(TOKEN_KEY); + if (cancelled) return; + const message = err instanceof Error ? err.message : ""; + const looksLikeAuthFailure = /unauthor|verify token|401/i.test(message); + if (looksLikeAuthFailure) { + localStorage.removeItem(config.tokenKey); } retry(); }) @@ -120,7 +168,87 @@ export function useMcpCallLogs() { connRef.current = null; } }; - }, []); + }, [binding]); return { rows, connectionState }; } + +const mcpBinding: TableBinding = { + tableName: "my_visible_mcp_call_log", + iter: (ctx) => ctx.db.myVisibleMcpCallLog.iter(), + onInsert: (conn, cb) => { + conn.db.myVisibleMcpCallLog.onInsert((_ctx: EventContext, row: McpCallLogRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.myVisibleMcpCallLog.onDelete((_ctx: EventContext, row: McpCallLogRow) => cb(row)); + }, + onUpdate: (conn, cb) => { + conn.db.myVisibleMcpCallLog.onUpdate((_ctx: EventContext, _old: McpCallLogRow, row: McpCallLogRow) => cb(row)); + }, +}; + +const aiQueryBinding: TableBinding = { + tableName: "my_visible_ai_query_log", + iter: (ctx) => ctx.db.myVisibleAiQueryLog.iter(), + onInsert: (conn, cb) => { + conn.db.myVisibleAiQueryLog.onInsert((_ctx: EventContext, row: AiQueryLogRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.myVisibleAiQueryLog.onDelete((_ctx: EventContext, row: AiQueryLogRow) => cb(row)); + }, + onUpdate: (conn, cb) => { + conn.db.myVisibleAiQueryLog.onUpdate((_ctx: EventContext, _old: AiQueryLogRow, row: AiQueryLogRow) => cb(row)); + }, +}; + +const publishedQaBinding: TableBinding = { + tableName: "published_qa", + iter: (ctx) => ctx.db.publishedQa.iter(), + onInsert: (conn, cb) => { + conn.db.publishedQa.onInsert((_ctx: EventContext, row: PublishedQaRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.publishedQa.onDelete((_ctx: EventContext, row: PublishedQaRow) => cb(row)); + }, +}; + +const qaEntriesBinding: TableBinding = { + tableName: "my_visible_qa_entries", + iter: (ctx) => ctx.db.myVisibleQaEntries.iter(), + onInsert: (conn, cb) => { + conn.db.myVisibleQaEntries.onInsert((_ctx: EventContext, row: QaEntriesRow) => cb(row)); + }, + onDelete: (conn, cb) => { + conn.db.myVisibleQaEntries.onDelete((_ctx: EventContext, row: QaEntriesRow) => cb(row)); + }, + onUpdate: (conn, cb) => { + conn.db.myVisibleQaEntries.onUpdate((_ctx: EventContext, _old: QaEntriesRow, row: QaEntriesRow) => cb(row)); + }, +}; + +export function useMcpCallLogs(ensureEnrolled?: EnsureEnrolled) { + return useTableSubscription(mcpBinding, ensureEnrolled); +} + +export function useAiQueryLogs(ensureEnrolled?: EnsureEnrolled) { + return useTableSubscription(aiQueryBinding, ensureEnrolled); +} + +/** + * Public — no enrollment required. Backed by the `published_qa` anonymousView, + * which returns only rows reviewers have explicitly published. Safe to call + * from unauthenticated pages. + */ +export function usePublishedQa() { + return useTableSubscription(publishedQaBinding); +} + +/** + * Reviewer-only. Subscribes to the curated Q&A entries table (separate from + * mcp_call_log telemetry). Use this for the editorial surface — every row + * here is a Q&A pair, either tied to a real MCP call (sourceMcpCorrelationId + * non-null) or a manual entry (null). + */ +export function useQaEntries(ensureEnrolled?: EnsureEnrolled) { + return useTableSubscription(qaEntriesBinding, ensureEnrolled); +} diff --git a/apps/internal-tool/src/lib/env.ts b/apps/internal-tool/src/lib/env.ts new file mode 100644 index 0000000000..ff3aafc4bf --- /dev/null +++ b/apps/internal-tool/src/lib/env.ts @@ -0,0 +1,18 @@ +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; + +const IS_DEV = process.env.NODE_ENV === "development"; +const PLACEHOLDER = "REPLACE_ME"; + +/** + * In dev, fall back to a seeded local default when an env var is missing or + * still holds the `REPLACE_ME` placeholder. In prod, missing/placeholder values + * are a deployment misconfiguration and throw immediately so requests don't + * silently go out with empty auth headers or a blank base URL. + */ +export function envOrDevDefault(value: string | undefined, devDefault: string, name: string): string { + if (!value || value === PLACEHOLDER) { + if (IS_DEV) return devDefault; + throw new StackAssertionError(`${name} is not configured. Set the NEXT_PUBLIC_STACK_* vars in .env.local or the hosting platform env.`); + } + return value; +} diff --git a/apps/internal-tool/src/lib/mcp-review-api.ts b/apps/internal-tool/src/lib/mcp-review-api.ts index de916b3acf..d943faf4fe 100644 --- a/apps/internal-tool/src/lib/mcp-review-api.ts +++ b/apps/internal-tool/src/lib/mcp-review-api.ts @@ -1,19 +1,12 @@ -const IS_DEV = process.env.NODE_ENV === "development"; -const PLACEHOLDER = "REPLACE_ME"; - -function envOrDevDefault(value: string | undefined, devDefault: string): string { - if (!value || value === PLACEHOLDER) { - return IS_DEV ? devDefault : ""; - } - return value; -} +import { envOrDevDefault } from "./env"; const PORT_PREFIX = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX ?? "81"; -const API_URL = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_API_URL, `http://localhost:${PORT_PREFIX}02`); -const PROJECT_ID = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_PROJECT_ID, "internal"); +const API_URL = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_API_URL, `http://localhost:${PORT_PREFIX}02`, "NEXT_PUBLIC_STACK_API_URL"); +const PROJECT_ID = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_PROJECT_ID, "internal", "NEXT_PUBLIC_STACK_PROJECT_ID"); const PUBLISHABLE_CLIENT_KEY = envOrDevDefault( process.env.NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY, "this-publishable-client-key-is-for-local-development-only", + "NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY", ); async function post(path: string, body: unknown, authHeaders: Record): Promise { @@ -39,6 +32,9 @@ export function makeMcpReviewApi(authHeaders: Record) { markReviewed: (body: { correlationId: string }) => post("mark-reviewed", body, authHeaders), + unmarkReviewed: (body: { correlationId: string }) => + post("unmark-reviewed", body, authHeaders), + updateCorrection: (body: { correlationId: string; correctedQuestion: string; @@ -52,7 +48,35 @@ export function makeMcpReviewApi(authHeaders: Record) { publish: boolean; }) => post("add-manual", body, authHeaders), - delete: (body: { correlationId: string }) => + updateQaEntry: (body: { + qaId: string; + question: string; + answer: string; + publish: boolean; + }) => post("update-qa-entry", body, authHeaders), + + delete: (body: { qaId: string }) => post("delete", body, authHeaders), }; } + +export async function enrollSpacetimeReviewer( + body: { identity: string }, + authHeaders: Record, +): Promise { + const res = await fetch(`${API_URL}/api/latest/internal/spacetimedb-enroll-reviewer`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-stack-access-type": "client", + "x-stack-project-id": PROJECT_ID, + "x-stack-publishable-client-key": PUBLISHABLE_CLIENT_KEY, + ...authHeaders, + }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text(); + throw new Error(`SpacetimeDB enroll error (${res.status}): ${text}`); + } +} diff --git a/apps/internal-tool/src/module_bindings/add_manual_qa_reducer.ts b/apps/internal-tool/src/module_bindings/add_manual_qa_reducer.ts index 97d707b5c7..ce711ebca0 100644 --- a/apps/internal-tool/src/module_bindings/add_manual_qa_reducer.ts +++ b/apps/internal-tool/src/module_bindings/add_manual_qa_reducer.ts @@ -15,5 +15,5 @@ export default { question: __t.string(), answer: __t.string(), publish: __t.bool(), - reviewedBy: __t.string(), + createdBy: __t.string(), }; diff --git a/apps/internal-tool/src/module_bindings/add_operator_reducer.ts b/apps/internal-tool/src/module_bindings/add_operator_reducer.ts new file mode 100644 index 0000000000..11cc2cf43f --- /dev/null +++ b/apps/internal-tool/src/module_bindings/add_operator_reducer.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + identity: __t.identity(), + stackUserId: __t.string(), + displayName: __t.string(), +}; diff --git a/apps/internal-tool/src/module_bindings/backfill_qa_entries_reducer.ts b/apps/internal-tool/src/module_bindings/backfill_qa_entries_reducer.ts new file mode 100644 index 0000000000..eb8e28c412 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/backfill_qa_entries_reducer.ts @@ -0,0 +1,15 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts b/apps/internal-tool/src/module_bindings/delete_ai_query_log_reducer.ts similarity index 100% rename from apps/backend/src/lib/ai/spacetimedb-bindings/delete_qa_entry_reducer.ts rename to apps/internal-tool/src/module_bindings/delete_ai_query_log_reducer.ts diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts b/apps/internal-tool/src/module_bindings/delete_mcp_call_log_reducer.ts similarity index 93% rename from apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts rename to apps/internal-tool/src/module_bindings/delete_mcp_call_log_reducer.ts index 87a1e7d923..1fa706ca53 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/mark_human_reviewed_reducer.ts +++ b/apps/internal-tool/src/module_bindings/delete_mcp_call_log_reducer.ts @@ -13,5 +13,4 @@ import { export default { token: __t.string(), correlationId: __t.string(), - reviewedBy: __t.string(), }; diff --git a/apps/internal-tool/src/module_bindings/delete_qa_entry_reducer.ts b/apps/internal-tool/src/module_bindings/delete_qa_entry_reducer.ts index 1fa706ca53..336a5109bc 100644 --- a/apps/internal-tool/src/module_bindings/delete_qa_entry_reducer.ts +++ b/apps/internal-tool/src/module_bindings/delete_qa_entry_reducer.ts @@ -12,5 +12,5 @@ import { export default { token: __t.string(), - correlationId: __t.string(), + qaId: __t.u64(), }; diff --git a/apps/internal-tool/src/module_bindings/enroll_service_reducer.ts b/apps/internal-tool/src/module_bindings/enroll_service_reducer.ts new file mode 100644 index 0000000000..ca2c40e7d2 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/enroll_service_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + displayName: __t.string(), +}; diff --git a/apps/internal-tool/src/module_bindings/index.ts b/apps/internal-tool/src/module_bindings/index.ts index bf16c63e07..ccf68258e7 100644 --- a/apps/internal-tool/src/module_bindings/index.ts +++ b/apps/internal-tool/src/module_bindings/index.ts @@ -35,42 +35,98 @@ import { // Import all reducer arg schemas import AddManualQaReducer from "./add_manual_qa_reducer"; +import AddOperatorReducer from "./add_operator_reducer"; +import BackfillQaEntriesReducer from "./backfill_qa_entries_reducer"; +import DeleteAiQueryLogReducer from "./delete_ai_query_log_reducer"; +import DeleteMcpCallLogReducer from "./delete_mcp_call_log_reducer"; import DeleteQaEntryReducer from "./delete_qa_entry_reducer"; +import EnrollServiceReducer from "./enroll_service_reducer"; +import LogAiQueryReducer from "./log_ai_query_reducer"; import LogMcpCallReducer from "./log_mcp_call_reducer"; import MarkHumanReviewedReducer from "./mark_human_reviewed_reducer"; -import UpdateHumanCorrectionReducer from "./update_human_correction_reducer"; +import RemoveOperatorReducer from "./remove_operator_reducer"; +import SetQaPublishedReducer from "./set_qa_published_reducer"; +import UnmarkHumanReviewedReducer from "./unmark_human_reviewed_reducer"; +import UpdateAiQueryCostReducer from "./update_ai_query_cost_reducer"; import UpdateMcpQaReviewReducer from "./update_mcp_qa_review_reducer"; +import UpdateQaEntryReducer from "./update_qa_entry_reducer"; +import UpdateQaEntryWithPublishReducer from "./update_qa_entry_with_publish_reducer"; +import UpsertQaFromCallReducer from "./upsert_qa_from_call_reducer"; // Import all procedure arg schemas // Import all table schema definitions -import McpCallLogRow from "./mcp_call_log_table"; +import MyVisibleAiQueryLogRow from "./my_visible_ai_query_log_table"; +import MyVisibleMcpCallLogRow from "./my_visible_mcp_call_log_table"; +import MyVisibleQaEntriesRow from "./my_visible_qa_entries_table"; +import OperatorsRow from "./operators_table"; +import PublishedQaRow from "./published_qa_table"; /** Type-only namespace exports for generated type groups. */ /** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ const tablesSchema = __schema({ - mcpCallLog: __table({ - name: 'mcp_call_log', + operators: __table({ + name: 'operators', indexes: [ - { accessor: 'id', name: 'mcp_call_log_id_idx_btree', algorithm: 'btree', columns: [ - 'id', + { accessor: 'identity', name: 'operators_identity_idx_btree', algorithm: 'btree', columns: [ + 'identity', ] }, ], constraints: [ - { name: 'mcp_call_log_id_key', constraint: 'unique', columns: ['id'] }, + { name: 'operators_identity_key', constraint: 'unique', columns: ['identity'] }, ], - }, McpCallLogRow), + }, OperatorsRow), + myVisibleAiQueryLog: __table({ + name: 'my_visible_ai_query_log', + indexes: [ + ], + constraints: [ + ], + }, MyVisibleAiQueryLogRow), + myVisibleMcpCallLog: __table({ + name: 'my_visible_mcp_call_log', + indexes: [ + ], + constraints: [ + ], + }, MyVisibleMcpCallLogRow), + myVisibleQaEntries: __table({ + name: 'my_visible_qa_entries', + indexes: [ + ], + constraints: [ + ], + }, MyVisibleQaEntriesRow), + publishedQa: __table({ + name: 'published_qa', + indexes: [ + ], + constraints: [ + ], + }, PublishedQaRow), }); /** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ const reducersSchema = __reducers( __reducerSchema("add_manual_qa", AddManualQaReducer), + __reducerSchema("add_operator", AddOperatorReducer), + __reducerSchema("backfill_qa_entries", BackfillQaEntriesReducer), + __reducerSchema("delete_ai_query_log", DeleteAiQueryLogReducer), + __reducerSchema("delete_mcp_call_log", DeleteMcpCallLogReducer), __reducerSchema("delete_qa_entry", DeleteQaEntryReducer), + __reducerSchema("enroll_service", EnrollServiceReducer), + __reducerSchema("log_ai_query", LogAiQueryReducer), __reducerSchema("log_mcp_call", LogMcpCallReducer), __reducerSchema("mark_human_reviewed", MarkHumanReviewedReducer), - __reducerSchema("update_human_correction", UpdateHumanCorrectionReducer), + __reducerSchema("remove_operator", RemoveOperatorReducer), + __reducerSchema("set_qa_published", SetQaPublishedReducer), + __reducerSchema("unmark_human_reviewed", UnmarkHumanReviewedReducer), + __reducerSchema("update_ai_query_cost", UpdateAiQueryCostReducer), __reducerSchema("update_mcp_qa_review", UpdateMcpQaReviewReducer), + __reducerSchema("update_qa_entry", UpdateQaEntryReducer), + __reducerSchema("update_qa_entry_with_publish", UpdateQaEntryWithPublishReducer), + __reducerSchema("upsert_qa_from_call", UpsertQaFromCallReducer), ); /** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ diff --git a/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts b/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts new file mode 100644 index 0000000000..e3793d29f2 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/log_ai_query_reducer.ts @@ -0,0 +1,39 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + cacheCreationTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + cacheDiscountUsd: __t.option(__t.f64()), + openrouterGenerationId: __t.option(__t.string()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}; diff --git a/apps/internal-tool/src/module_bindings/my_visible_ai_query_log_table.ts b/apps/internal-tool/src/module_bindings/my_visible_ai_query_log_table.ts new file mode 100644 index 0000000000..37460d0392 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/my_visible_ai_query_log_table.ts @@ -0,0 +1,41 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64(), + shard: __t.u8(), + correlationId: __t.string().name("correlation_id"), + createdAt: __t.timestamp().name("created_at"), + mode: __t.string(), + systemPromptId: __t.string().name("system_prompt_id"), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string().name("model_id"), + isAuthenticated: __t.bool().name("is_authenticated"), + projectId: __t.option(__t.string()).name("project_id"), + userId: __t.option(__t.string()).name("user_id"), + requestedToolsJson: __t.string().name("requested_tools_json"), + messagesJson: __t.string().name("messages_json"), + stepsJson: __t.string().name("steps_json"), + finalText: __t.string().name("final_text"), + inputTokens: __t.option(__t.u32()).name("input_tokens"), + outputTokens: __t.option(__t.u32()).name("output_tokens"), + cachedInputTokens: __t.option(__t.u32()).name("cached_input_tokens"), + cacheCreationTokens: __t.option(__t.u32()).name("cache_creation_tokens"), + costUsd: __t.option(__t.f64()).name("cost_usd"), + cacheDiscountUsd: __t.option(__t.f64()).name("cache_discount_usd"), + openrouterGenerationId: __t.option(__t.string()).name("openrouter_generation_id"), + stepCount: __t.u32().name("step_count"), + durationMs: __t.u64().name("duration_ms"), + errorMessage: __t.option(__t.string()).name("error_message"), + conversationId: __t.option(__t.string()).name("conversation_id"), +}); diff --git a/apps/internal-tool/src/module_bindings/mcp_call_log_table.ts b/apps/internal-tool/src/module_bindings/my_visible_mcp_call_log_table.ts similarity index 95% rename from apps/internal-tool/src/module_bindings/mcp_call_log_table.ts rename to apps/internal-tool/src/module_bindings/my_visible_mcp_call_log_table.ts index bcf0dab879..cd54c54f7b 100644 --- a/apps/internal-tool/src/module_bindings/mcp_call_log_table.ts +++ b/apps/internal-tool/src/module_bindings/my_visible_mcp_call_log_table.ts @@ -11,7 +11,8 @@ import { } from "spacetimedb"; export default __t.row({ - id: __t.u64().primaryKey(), + id: __t.u64(), + shard: __t.u8(), correlationId: __t.string().name("correlation_id"), conversationId: __t.option(__t.string()).name("conversation_id"), createdAt: __t.timestamp().name("created_at"), @@ -39,6 +40,6 @@ export default __t.row({ humanReviewedBy: __t.option(__t.string()).name("human_reviewed_by"), humanCorrectedQuestion: __t.option(__t.string()).name("human_corrected_question"), humanCorrectedAnswer: __t.option(__t.string()).name("human_corrected_answer"), - publishedToQa: __t.option(__t.bool()).name("published_to_qa"), + publishedToQa: __t.bool().name("published_to_qa"), publishedAt: __t.option(__t.timestamp()).name("published_at"), }); diff --git a/apps/internal-tool/src/module_bindings/my_visible_qa_entries_table.ts b/apps/internal-tool/src/module_bindings/my_visible_qa_entries_table.ts new file mode 100644 index 0000000000..bf6877f2ee --- /dev/null +++ b/apps/internal-tool/src/module_bindings/my_visible_qa_entries_table.ts @@ -0,0 +1,26 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64(), + shard: __t.u8(), + sourceMcpCorrelationId: __t.option(__t.string()).name("source_mcp_correlation_id"), + question: __t.string(), + answer: __t.string(), + createdBy: __t.string().name("created_by"), + createdAt: __t.timestamp().name("created_at"), + lastEditedBy: __t.string().name("last_edited_by"), + lastEditedAt: __t.timestamp().name("last_edited_at"), + published: __t.bool(), + firstPublishedAt: __t.option(__t.timestamp()).name("first_published_at"), + lastPublishedAt: __t.option(__t.timestamp()).name("last_published_at"), +}); diff --git a/apps/internal-tool/src/module_bindings/operators_table.ts b/apps/internal-tool/src/module_bindings/operators_table.ts new file mode 100644 index 0000000000..9f5a0d22f7 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/operators_table.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + identity: __t.identity().primaryKey(), + addedAt: __t.timestamp().name("added_at"), + stackUserId: __t.string().name("stack_user_id"), + displayName: __t.string().name("display_name"), +}); diff --git a/apps/internal-tool/src/module_bindings/published_qa_table.ts b/apps/internal-tool/src/module_bindings/published_qa_table.ts new file mode 100644 index 0000000000..3d5f5bf7e1 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/published_qa_table.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64(), + question: __t.string(), + answer: __t.string(), + publishedAt: __t.option(__t.timestamp()).name("published_at"), +}); diff --git a/apps/internal-tool/src/module_bindings/remove_operator_reducer.ts b/apps/internal-tool/src/module_bindings/remove_operator_reducer.ts new file mode 100644 index 0000000000..0dfadd7bdd --- /dev/null +++ b/apps/internal-tool/src/module_bindings/remove_operator_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + identity: __t.identity(), +}; diff --git a/apps/internal-tool/src/module_bindings/set_qa_published_reducer.ts b/apps/internal-tool/src/module_bindings/set_qa_published_reducer.ts new file mode 100644 index 0000000000..33e90cb5c3 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/set_qa_published_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + qaId: __t.u64(), + publish: __t.bool(), +}; diff --git a/apps/internal-tool/src/module_bindings/types.ts b/apps/internal-tool/src/module_bindings/types.ts index 4af9e7b736..572502ea94 100644 --- a/apps/internal-tool/src/module_bindings/types.ts +++ b/apps/internal-tool/src/module_bindings/types.ts @@ -10,8 +10,40 @@ import { type Infer as __Infer, } from "spacetimedb"; +export const AiQueryLog = __t.object("AiQueryLog", { + id: __t.u64(), + shard: __t.u8(), + correlationId: __t.string(), + createdAt: __t.timestamp(), + mode: __t.string(), + systemPromptId: __t.string(), + quality: __t.string(), + speed: __t.string(), + modelId: __t.string(), + isAuthenticated: __t.bool(), + projectId: __t.option(__t.string()), + userId: __t.option(__t.string()), + requestedToolsJson: __t.string(), + messagesJson: __t.string(), + stepsJson: __t.string(), + finalText: __t.string(), + inputTokens: __t.option(__t.u32()), + outputTokens: __t.option(__t.u32()), + cachedInputTokens: __t.option(__t.u32()), + cacheCreationTokens: __t.option(__t.u32()), + costUsd: __t.option(__t.f64()), + cacheDiscountUsd: __t.option(__t.f64()), + openrouterGenerationId: __t.option(__t.string()), + stepCount: __t.u32(), + durationMs: __t.u64(), + errorMessage: __t.option(__t.string()), + conversationId: __t.option(__t.string()), +}); +export type AiQueryLog = __Infer; + export const McpCallLog = __t.object("McpCallLog", { id: __t.u64(), + shard: __t.u8(), correlationId: __t.string(), conversationId: __t.option(__t.string()), createdAt: __t.timestamp(), @@ -39,8 +71,52 @@ export const McpCallLog = __t.object("McpCallLog", { humanReviewedBy: __t.option(__t.string()), humanCorrectedQuestion: __t.option(__t.string()), humanCorrectedAnswer: __t.option(__t.string()), - publishedToQa: __t.option(__t.bool()), + publishedToQa: __t.bool(), publishedAt: __t.option(__t.timestamp()), }); export type McpCallLog = __Infer; +export const MyVisibleAiQueryLog = __t.object("MyVisibleAiQueryLog", {}); +export type MyVisibleAiQueryLog = __Infer; + +export const MyVisibleMcpCallLog = __t.object("MyVisibleMcpCallLog", {}); +export type MyVisibleMcpCallLog = __Infer; + +export const MyVisibleQaEntries = __t.object("MyVisibleQaEntries", {}); +export type MyVisibleQaEntries = __Infer; + +export const Operators = __t.object("Operators", { + identity: __t.identity(), + addedAt: __t.timestamp(), + stackUserId: __t.string(), + displayName: __t.string(), +}); +export type Operators = __Infer; + +export const PublishedQa = __t.object("PublishedQa", {}); +export type PublishedQa = __Infer; + +export const PublishedQaRow = __t.object("PublishedQaRow", { + id: __t.u64(), + question: __t.string(), + answer: __t.string(), + publishedAt: __t.option(__t.timestamp()), +}); +export type PublishedQaRow = __Infer; + +export const QaEntries = __t.object("QaEntries", { + id: __t.u64(), + shard: __t.u8(), + sourceMcpCorrelationId: __t.option(__t.string()), + question: __t.string(), + answer: __t.string(), + createdBy: __t.string(), + createdAt: __t.timestamp(), + lastEditedBy: __t.string(), + lastEditedAt: __t.timestamp(), + published: __t.bool(), + firstPublishedAt: __t.option(__t.timestamp()), + lastPublishedAt: __t.option(__t.timestamp()), +}); +export type QaEntries = __Infer; + diff --git a/apps/internal-tool/src/module_bindings/types/reducers.ts b/apps/internal-tool/src/module_bindings/types/reducers.ts index 87a6606ae4..6df999db6f 100644 --- a/apps/internal-tool/src/module_bindings/types/reducers.ts +++ b/apps/internal-tool/src/module_bindings/types/reducers.ts @@ -7,16 +7,40 @@ import { type Infer as __Infer } from "spacetimedb"; // Import all reducer arg schemas import AddManualQaReducer from "../add_manual_qa_reducer"; +import AddOperatorReducer from "../add_operator_reducer"; +import BackfillQaEntriesReducer from "../backfill_qa_entries_reducer"; +import DeleteAiQueryLogReducer from "../delete_ai_query_log_reducer"; +import DeleteMcpCallLogReducer from "../delete_mcp_call_log_reducer"; import DeleteQaEntryReducer from "../delete_qa_entry_reducer"; +import EnrollServiceReducer from "../enroll_service_reducer"; +import LogAiQueryReducer from "../log_ai_query_reducer"; import LogMcpCallReducer from "../log_mcp_call_reducer"; import MarkHumanReviewedReducer from "../mark_human_reviewed_reducer"; -import UpdateHumanCorrectionReducer from "../update_human_correction_reducer"; +import RemoveOperatorReducer from "../remove_operator_reducer"; +import SetQaPublishedReducer from "../set_qa_published_reducer"; +import UnmarkHumanReviewedReducer from "../unmark_human_reviewed_reducer"; +import UpdateAiQueryCostReducer from "../update_ai_query_cost_reducer"; import UpdateMcpQaReviewReducer from "../update_mcp_qa_review_reducer"; +import UpdateQaEntryReducer from "../update_qa_entry_reducer"; +import UpdateQaEntryWithPublishReducer from "../update_qa_entry_with_publish_reducer"; +import UpsertQaFromCallReducer from "../upsert_qa_from_call_reducer"; export type AddManualQaParams = __Infer; +export type AddOperatorParams = __Infer; +export type BackfillQaEntriesParams = __Infer; +export type DeleteAiQueryLogParams = __Infer; +export type DeleteMcpCallLogParams = __Infer; export type DeleteQaEntryParams = __Infer; +export type EnrollServiceParams = __Infer; +export type LogAiQueryParams = __Infer; export type LogMcpCallParams = __Infer; export type MarkHumanReviewedParams = __Infer; -export type UpdateHumanCorrectionParams = __Infer; +export type RemoveOperatorParams = __Infer; +export type SetQaPublishedParams = __Infer; +export type UnmarkHumanReviewedParams = __Infer; +export type UpdateAiQueryCostParams = __Infer; export type UpdateMcpQaReviewParams = __Infer; +export type UpdateQaEntryParams = __Infer; +export type UpdateQaEntryWithPublishParams = __Infer; +export type UpsertQaFromCallParams = __Infer; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts b/apps/internal-tool/src/module_bindings/unmark_human_reviewed_reducer.ts similarity index 77% rename from apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts rename to apps/internal-tool/src/module_bindings/unmark_human_reviewed_reducer.ts index 2ae110a192..1fa706ca53 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/update_human_correction_reducer.ts +++ b/apps/internal-tool/src/module_bindings/unmark_human_reviewed_reducer.ts @@ -13,8 +13,4 @@ import { export default { token: __t.string(), correlationId: __t.string(), - correctedQuestion: __t.string(), - correctedAnswer: __t.string(), - publish: __t.bool(), - reviewedBy: __t.string(), }; diff --git a/apps/internal-tool/src/module_bindings/update_ai_query_cost_reducer.ts b/apps/internal-tool/src/module_bindings/update_ai_query_cost_reducer.ts new file mode 100644 index 0000000000..5bbb7e20bb --- /dev/null +++ b/apps/internal-tool/src/module_bindings/update_ai_query_cost_reducer.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + correlationId: __t.string(), + costUsd: __t.option(__t.f64()), + cacheDiscountUsd: __t.option(__t.f64()), +}; diff --git a/apps/internal-tool/src/module_bindings/update_qa_entry_reducer.ts b/apps/internal-tool/src/module_bindings/update_qa_entry_reducer.ts new file mode 100644 index 0000000000..a2c44fc194 --- /dev/null +++ b/apps/internal-tool/src/module_bindings/update_qa_entry_reducer.ts @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + token: __t.string(), + qaId: __t.u64(), + question: __t.string(), + answer: __t.string(), + editedBy: __t.string(), +}; diff --git a/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts b/apps/internal-tool/src/module_bindings/update_qa_entry_with_publish_reducer.ts similarity index 91% rename from apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts rename to apps/internal-tool/src/module_bindings/update_qa_entry_with_publish_reducer.ts index 97d707b5c7..c78aa1b05d 100644 --- a/apps/backend/src/lib/ai/spacetimedb-bindings/add_manual_qa_reducer.ts +++ b/apps/internal-tool/src/module_bindings/update_qa_entry_with_publish_reducer.ts @@ -12,8 +12,9 @@ import { export default { token: __t.string(), + qaId: __t.u64(), question: __t.string(), answer: __t.string(), publish: __t.bool(), - reviewedBy: __t.string(), + editedBy: __t.string(), }; diff --git a/apps/internal-tool/src/module_bindings/update_human_correction_reducer.ts b/apps/internal-tool/src/module_bindings/upsert_qa_from_call_reducer.ts similarity index 81% rename from apps/internal-tool/src/module_bindings/update_human_correction_reducer.ts rename to apps/internal-tool/src/module_bindings/upsert_qa_from_call_reducer.ts index 2ae110a192..e608716e8a 100644 --- a/apps/internal-tool/src/module_bindings/update_human_correction_reducer.ts +++ b/apps/internal-tool/src/module_bindings/upsert_qa_from_call_reducer.ts @@ -13,8 +13,8 @@ import { export default { token: __t.string(), correlationId: __t.string(), - correctedQuestion: __t.string(), - correctedAnswer: __t.string(), + question: __t.string(), + answer: __t.string(), publish: __t.bool(), - reviewedBy: __t.string(), + editedBy: __t.string(), }; diff --git a/apps/internal-tool/src/stack.ts b/apps/internal-tool/src/stack.ts index 7de289ad98..ab9edf5eda 100644 --- a/apps/internal-tool/src/stack.ts +++ b/apps/internal-tool/src/stack.ts @@ -1,26 +1,15 @@ import { StackClientApp } from "@stackframe/stack"; - -const IS_DEV = process.env.NODE_ENV === "development"; -const PLACEHOLDER = "REPLACE_ME"; - -// In dev, fall back to the seeded "internal" project if env vars are placeholders. -// In prod, the real values must be set via hosting platform env vars. -function envOrDevDefault(value: string | undefined, devDefault: string): string { - if (!value || value === PLACEHOLDER) { - if (IS_DEV) return devDefault; - throw new Error("Stack Auth env var is not configured. Set the NEXT_PUBLIC_STACK_* vars in .env.local or hosting platform env."); - } - return value; -} +import { envOrDevDefault } from "./lib/env"; const portPrefix = process.env.NEXT_PUBLIC_STACK_PORT_PREFIX ?? "81"; -const projectId = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_PROJECT_ID, "internal"); +const projectId = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_PROJECT_ID, "internal", "NEXT_PUBLIC_STACK_PROJECT_ID"); const publishableClientKey = envOrDevDefault( process.env.NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY, "this-publishable-client-key-is-for-local-development-only", + "NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY", ); -const apiUrl = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_API_URL, `http://localhost:${portPrefix}02`); +const apiUrl = envOrDevDefault(process.env.NEXT_PUBLIC_STACK_API_URL, `http://localhost:${portPrefix}02`, "NEXT_PUBLIC_STACK_API_URL"); export const stackClientApp = new StackClientApp({ projectId, diff --git a/apps/internal-tool/src/types.ts b/apps/internal-tool/src/types.ts index 50f0320704..dfbaaf064c 100644 --- a/apps/internal-tool/src/types.ts +++ b/apps/internal-tool/src/types.ts @@ -1 +1,6 @@ -export type { McpCallLog as McpCallLogRow } from "./module_bindings/types"; +export type { + AiQueryLog as AiQueryLogRow, + McpCallLog as McpCallLogRow, + PublishedQaRow, + QaEntries as QaEntriesRow, +} from "./module_bindings/types"; diff --git a/packages/stack-shared/src/ai/image-limits.ts b/packages/stack-shared/src/ai/image-limits.ts index 687b8af8c0..9068aee681 100644 --- a/packages/stack-shared/src/ai/image-limits.ts +++ b/packages/stack-shared/src/ai/image-limits.ts @@ -15,24 +15,33 @@ export function estimateBase64ByteLength(dataUrl: string): number { return Math.max(0, Math.floor((base64.length * 3) / 4) - padding); } -type ValidationResult = { ok: true } | { ok: false, reason: string }; +export type ImageValidationFailure = + | { code: "too_many", maxImages: number } + | { code: "too_large", maxBytes: number, actualBytes: number }; + +export type ImageValidationResult = + | { ok: true } + | { ok: false, failure: ImageValidationFailure, reason: string }; + type UnknownPart = { type?: unknown, image?: unknown }; type MessageLike = { role?: unknown, content?: unknown }; -export function validateImageCount(imageCount: number): ValidationResult { +export function validateImageCount(imageCount: number): ImageValidationResult { if (imageCount > MAX_IMAGES_PER_MESSAGE) { return { ok: false, + failure: { code: "too_many", maxImages: MAX_IMAGES_PER_MESSAGE }, reason: `Maximum ${MAX_IMAGES_PER_MESSAGE} images per message.`, }; } return { ok: true }; } -export function validateImageByteLength(bytes: number): ValidationResult { +export function validateImageByteLength(bytes: number): ImageValidationResult { if (bytes > MAX_IMAGE_BYTES_PER_FILE) { return { ok: false, + failure: { code: "too_large", maxBytes: MAX_IMAGE_BYTES_PER_FILE, actualBytes: bytes }, reason: `Image exceeds ${MAX_IMAGE_MB_PER_FILE}MB limit (${(bytes / 1024 / 1024).toFixed(2)}MB).`, }; } @@ -40,7 +49,7 @@ export function validateImageByteLength(bytes: number): ValidationResult { } /** Validates per-message image count and per-file size for user messages. */ -export function validateImageAttachments(messages: readonly MessageLike[]): ValidationResult { +export function validateImageAttachments(messages: readonly MessageLike[]): ImageValidationResult { for (const msg of messages) { if (!Array.isArray(msg.content)) continue; let imageCount = 0; diff --git a/packages/stack-shared/src/known-errors.tsx b/packages/stack-shared/src/known-errors.tsx index e65a64bb4e..0ae8f8922d 100644 --- a/packages/stack-shared/src/known-errors.tsx +++ b/packages/stack-shared/src/known-errors.tsx @@ -1852,6 +1852,33 @@ const NewPurchasesBlocked = createKnownErrorConstructor( () => [] as const, ); +const TooManyImageAttachments = createKnownErrorConstructor( + KnownError, + "TOO_MANY_IMAGE_ATTACHMENTS", + (maxImages: number) => [ + 400, + `Maximum ${maxImages} images per message.`, + { + max_images: maxImages, + }, + ] as const, + (json) => [json.max_images] as const, +); + +const ImageAttachmentTooLarge = createKnownErrorConstructor( + KnownError, + "IMAGE_ATTACHMENT_TOO_LARGE", + (maxBytes: number, actualBytes: number) => [ + 400, + `Image exceeds ${maxBytes / (1024 * 1024)}MB limit (${(actualBytes / 1024 / 1024).toFixed(2)}MB).`, + { + max_bytes: maxBytes, + actual_bytes: actualBytes, + }, + ] as const, + (json) => [json.max_bytes, json.actual_bytes] as const, +); + export type KnownErrors = { [K in keyof typeof KnownErrors]: InstanceType; }; @@ -2000,6 +2027,8 @@ export const KnownErrors = { AnalyticsQueryTimeout, AnalyticsQueryError, AnalyticsNotEnabled, + TooManyImageAttachments, + ImageAttachmentTooLarge, } satisfies Record>;