custom events

apps/backend/.env.development

@@ -76,6 +76,10 @@ STACK_OPENAI_API_KEY=mock_openai_api_key
 STACK_STRIPE_SECRET_KEY=sk_test_mockstripekey
 STACK_STRIPE_WEBHOOK_SECRET=mock_stripe_webhook_secret
 STACK_OPENROUTER_API_KEY=FORWARD_TO_PRODUCTION
+# Optional: override docs origin for the `docs` AI tool bundle (defaults to http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}04 in dev, https://mcp.stack-auth.com in prod)
+# STACK_DOCS_INTERNAL_BASE_URL=http://localhost:8104
+# Optional: shared secret; when set, backend sends it and docs `/api/internal/docs-tools` requires it
+# STACK_INTERNAL_DOCS_TOOLS_SECRET=
 # Email monitor configuration for tests
 STACK_EMAIL_MONITOR_VERIFICATION_CALLBACK_URL=http://localhost:8101/handler/email-verification
 STACK_EMAIL_MONITOR_PROJECT_ID=internal

apps/backend/scripts/clickhouse-migrations.ts

@@ -17,23 +17,36 @@ export async function runClickhouseMigrations() {
   await client.exec({ query: USERS_TABLE_BASE_SQL });
   await client.exec({ query: USERS_VIEW_SQL });
   await client.exec({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL });
+  await client.exec({ query: EVENTS_ADD_EVENT_ID_COLUMNS_SQL });
   await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL });
   await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL });
   await client.exec({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL });
-  // Recreate the events view so SELECT * picks up columns added by EVENTS_ADD_REPLAY_COLUMNS_SQL
+  await client.exec({ query: SPANS_TABLE_SQL });
+  await client.exec({ query: SPANS_VIEW_SQL });
+  await client.exec({ query: EVENTS_ADD_FROM_SERVER_COLUMN_SQL });
+  await client.exec({ query: SPANS_ADD_FROM_SERVER_COLUMN_SQL });
+  await client.exec({ query: EVENTS_ADD_TRACE_ID_COLUMN_SQL });
+  await client.exec({ query: SPANS_ADD_TRACE_ID_COLUMN_SQL });
+  // Recreate the events view so SELECT * picks up columns added by migrations
   await client.exec({ query: EVENTS_VIEW_SQL });
+  // Recreate the spans view so SELECT * picks up columns added by migrations
+  await client.exec({ query: SPANS_VIEW_SQL });
   const queries = [
     "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;",
     "REVOKE ALL FROM limited_user;",
     "GRANT SELECT ON default.events TO limited_user;",
     "GRANT SELECT ON default.users TO limited_user;",
+    "GRANT SELECT ON default.spans TO limited_user;",
   ];
   await client.exec({
     query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
   });
   await client.exec({
     query: "CREATE ROW POLICY IF NOT EXISTS users_project_isolation ON default.users FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
   });
+  await client.exec({
+    query: "CREATE ROW POLICY IF NOT EXISTS spans_project_isolation ON default.spans FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
+  });
   for (const query of queries) {
     await client.exec({ query });
   }
@@ -50,15 +63,15 @@ CREATE TABLE IF NOT EXISTS analytics_internal.events (
     branch_id        String,
     user_id          Nullable(String),
     team_id          Nullable(String),
-    created_at DateTime64(3, 'UTC') DEFAULT now64(3)
+    created_at       DateTime64(3, 'UTC') DEFAULT now64(3)
 )
 ENGINE MergeTree
 PARTITION BY toYYYYMM(event_at)
 ORDER BY (project_id, branch_id, event_at);
 `;
 
 const EVENTS_VIEW_SQL = `
-CREATE OR REPLACE VIEW default.events 
+CREATE OR REPLACE VIEW default.events
 SQL SECURITY DEFINER
 AS
 SELECT *
@@ -146,7 +159,7 @@ ORDER BY (project_id, branch_id, id);
 `;
 
 const USERS_VIEW_SQL = `
-CREATE OR REPLACE VIEW default.users 
+CREATE OR REPLACE VIEW default.users
 SQL SECURITY DEFINER
 AS
 SELECT
@@ -197,6 +210,63 @@ WHERE event_type = '$token-refresh'
   AND data.refresh_token_id::Nullable(String) IS NOT NULL;
 `;
 
+const EVENTS_ADD_EVENT_ID_COLUMNS_SQL = `
+ALTER TABLE analytics_internal.events
+  ADD COLUMN IF NOT EXISTS event_id String DEFAULT '' AFTER event_type,
+  ADD COLUMN IF NOT EXISTS parent_span_ids Array(String) DEFAULT [] AFTER event_id;
+`;
+
+const SPANS_TABLE_SQL = `
+CREATE TABLE IF NOT EXISTS analytics_internal.spans (
+    span_type        LowCardinality(String),
+    span_id          String,
+    started_at       DateTime64(3, 'UTC'),
+    created_at       DateTime64(3, 'UTC') DEFAULT now64(3),
+    ended_at           Nullable(DateTime64(3, 'UTC')),
+    parent_ids       Array(String) DEFAULT [],
+    data             JSON,
+    project_id       String,
+    branch_id        String,
+    user_id          Nullable(String),
+    team_id          Nullable(String),
+    refresh_token_id Nullable(String),
+    session_replay_id Nullable(String),
+    session_replay_segment_id Nullable(String)
+)
+ENGINE ReplacingMergeTree(created_at)
+PARTITION BY toYYYYMM(started_at)
+ORDER BY (project_id, branch_id, span_id);
+`;
+
+const SPANS_VIEW_SQL = `
+CREATE OR REPLACE VIEW default.spans
+SQL SECURITY DEFINER
+AS
+SELECT *
+FROM analytics_internal.spans
+FINAL;
+`;
+
+const EVENTS_ADD_FROM_SERVER_COLUMN_SQL = `
+ALTER TABLE analytics_internal.events
+  ADD COLUMN IF NOT EXISTS from_server Bool DEFAULT false AFTER session_replay_segment_id;
+`;
+
+const SPANS_ADD_FROM_SERVER_COLUMN_SQL = `
+ALTER TABLE analytics_internal.spans
+  ADD COLUMN IF NOT EXISTS from_server Bool DEFAULT false AFTER session_replay_segment_id;
+`;
+
+const EVENTS_ADD_TRACE_ID_COLUMN_SQL = `
+ALTER TABLE analytics_internal.events
+  ADD COLUMN IF NOT EXISTS trace_id Nullable(String) AFTER event_id;
+`;
+
+const SPANS_ADD_TRACE_ID_COLUMN_SQL = `
+ALTER TABLE analytics_internal.spans
+  ADD COLUMN IF NOT EXISTS trace_id Nullable(String) AFTER span_id;
+`;
+
 const EXTERNAL_ANALYTICS_DB_SQL = `
 CREATE DATABASE IF NOT EXISTS analytics_internal;
 `;

apps/backend/src/app/api/latest/ai/query/[mode]/route.ts

@@ -129,7 +129,7 @@ export const POST = createSmartRouteHandler({
       return {
         statusCode: 200,
         bodyType: "json" as const,
-        body: { content: contentBlocks },
+        body: { content: contentBlocks, finalText: result.text },
       };
     }
   },

apps/backend/src/app/api/latest/analytics/events/batch/route.tsx

@@ -1,43 +1,20 @@
-import { getClickhouseAdminClient } from "@/lib/clickhouse";
+import { isValidClientAnalyticsEventType, isValidPublicAnalyticsEventType, PUBLIC_STACK_ANALYTICS_EVENT_TYPE_LIST, UUID_RE } from "@/lib/analytics-validation";
+import { insertAnalyticsEvents } from "@/lib/events";
 import { findRecentSessionReplay } from "@/lib/session-replays";
 import { getPrismaClientForTenancy } from "@/prisma-client";
 import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
 import { KnownErrors } from "@stackframe/stack-shared";
-import { adaptSchema, clientOrHigherAuthTypeSchema, yupArray, yupMixed, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
+import { adaptSchema, clientOrHigherAuthTypeSchema, yupArray, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
 import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
-
-const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+import { randomUUID } from "node:crypto";
 
 const MAX_EVENTS = 500;
 
-// Lone surrogates (\uD800-\uDFFF not part of a valid pair) are technically
-// representable in JS strings but rejected by ClickHouse's JSON parser.
-// The client-side event tracker can produce these when .substring() truncates
-// text in the middle of a surrogate pair (e.g. emoji characters).
-// eslint-disable-next-line no-control-regex
-const LONE_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
-
-function stripLoneSurrogates(value: unknown): unknown {
-  if (typeof value === "string") {
-    return value.replace(LONE_SURROGATE_RE, "\uFFFD");
-  }
-  if (Array.isArray(value)) {
-    return value.map(stripLoneSurrogates);
-  }
-  if (value !== null && typeof value === "object") {
-    return Object.fromEntries(
-      Object.entries(value).map(([k, v]) => [k, stripLoneSurrogates(v)])
-    );
-  }
-  return value;
-}
-
 export const POST = createSmartRouteHandler({
   metadata: {
     summary: "Upload analytics event batch",
-    description: "Uploads a batch of auto-captured analytics events ($page-view, $click).",
+    description: "Uploads a batch of Stack-managed browser analytics events or custom analytics events.",
     tags: ["Analytics Events"],
-    hidden: true,
   },
   request: yupObject({
     auth: yupObject({
@@ -47,14 +24,26 @@ export const POST = createSmartRouteHandler({
       refreshTokenId: adaptSchema,
     }).defined(),
     body: yupObject({
-      session_replay_segment_id: yupString().defined().matches(UUID_RE, "Invalid session_replay_segment_id"),
+      session_replay_id: yupString().optional().matches(UUID_RE, "Invalid session_replay_id"),
+      session_replay_segment_id: yupString().optional().matches(UUID_RE, "Invalid session_replay_segment_id"),
       batch_id: yupString().defined().matches(UUID_RE, "Invalid batch_id"),
       sent_at_ms: yupNumber().defined().integer().min(0),
       events: yupArray(
         yupObject({
-          event_type: yupString().defined().oneOf(["$page-view", "$click"]),
+          event_type: yupString().defined().test(
+            "analytics-event-type",
+            `event_type must be ${PUBLIC_STACK_ANALYTICS_EVENT_TYPE_LIST}, or a custom event name that does not start with "$" and only contains letters, numbers, ".", "_", ":", or "-"`,
+            (value) => isValidPublicAnalyticsEventType(value),
+          ),
           event_at_ms: yupNumber().defined().integer().min(0),
-          data: yupMixed().defined(),
+          data: yupObject({}).defined().unknown(true),
+          user_id: yupString().uuid().optional(),
+          team_id: yupString().uuid().optional(),
+          event_id: yupString().optional().matches(UUID_RE, "Invalid event_id"),
+          trace_id: yupString().optional().matches(UUID_RE, "Invalid trace_id"),
+          parent_span_ids: yupArray(yupString().defined().matches(UUID_RE, "Invalid parent_span_id")).optional().max(20),
+          session_replay_id: yupString().optional().matches(UUID_RE, "Invalid session_replay_id"),
+          session_replay_segment_id: yupString().optional().matches(UUID_RE, "Invalid session_replay_segment_id"),
         }).defined(),
       ).defined().min(1).max(MAX_EVENTS),
     }).defined(),
@@ -70,47 +59,101 @@ export const POST = createSmartRouteHandler({
     if (!auth.tenancy.config.apps.installed["analytics"]?.enabled) {
       throw new KnownErrors.AnalyticsNotEnabled();
     }
-    if (!auth.user) {
+
+    if (auth.type === "client" && !auth.user) {
       throw new KnownErrors.UserAuthenticationRequired();
     }
-    if (!auth.refreshTokenId) {
+    if (auth.type === "client" && !auth.refreshTokenId) {
       throw new StatusError(StatusError.BadRequest, "A refresh token is required for analytics events");
     }
+    if (auth.type === "client" && body.events.some((event) => event.user_id != null || event.team_id != null)) {
+      throw new StatusError(StatusError.BadRequest, "Client analytics events cannot override user_id or team_id");
+    }
+    if (auth.type === "client" && body.events.some((event) => !isValidClientAnalyticsEventType(event.event_type))) {
+      throw new StatusError(StatusError.BadRequest, "Client analytics events cannot use server-only event types");
+    }
 
     const projectId = auth.tenancy.project.id;
     const branchId = auth.tenancy.branchId;
-    const userId = auth.user.id;
-    const refreshTokenId = auth.refreshTokenId;
-    const tenancyId = auth.tenancy.id;
+    const clientUserId = auth.type === "client" ? auth.user?.id ?? null : null;
+    if (auth.type === "client" && clientUserId === null) {
+      throw new KnownErrors.UserAuthenticationRequired();
+    }
+    const defaultUserId = auth.user?.id ?? null;
 
-    const prisma = await getPrismaClientForTenancy(auth.tenancy);
-    const recentSession = await findRecentSessionReplay(prisma, { tenancyId, refreshTokenId });
+    const explicitSessionReplayIds = [...new Set([
+      body.session_replay_id,
+      ...body.events.map((event) => event.session_replay_id),
+    ].filter((sessionReplayId): sessionReplayId is string => sessionReplayId != null))];
 
-    const clickhouseClient = getClickhouseAdminClient();
+    const needsFallbackSessionReplayId = body.session_replay_id == null || body.events.some((event) => event.session_replay_id == null);
+    const shouldLoadPrisma = explicitSessionReplayIds.length > 0 || (auth.refreshTokenId != null && needsFallbackSessionReplayId);
+    const prisma = shouldLoadPrisma ? await getPrismaClientForTenancy(auth.tenancy) : null;
 
-    const rows = body.events.map((event) => ({
-      event_type: event.event_type,
-      event_at: new Date(event.event_at_ms),
-      data: stripLoneSurrogates(event.data),
-      project_id: projectId,
-      branch_id: branchId,
-      user_id: userId,
-      team_id: null,
-      refresh_token_id: refreshTokenId,
-      session_replay_id: recentSession?.id ?? null,
-      session_replay_segment_id: body.session_replay_segment_id,
-    }));
+    if (prisma && explicitSessionReplayIds.length > 0) {
+      const explicitSessionReplays = await prisma.sessionReplay.findMany({
+        where: {
+          tenancyId: auth.tenancy.id,
+          id: { in: explicitSessionReplayIds },
+        },
+        select: {
+          id: true,
+          refreshTokenId: true,
+        },
+      });
 
-    await clickhouseClient.insert({
-      table: "analytics_internal.events",
-      values: rows,
-      format: "JSONEachRow",
-      clickhouse_settings: {
-        date_time_input_format: "best_effort",
-        async_insert: 1,
-      },
+      const explicitSessionReplayIdsFound = new Set(explicitSessionReplays.map((sessionReplay) => sessionReplay.id));
+      const missingSessionReplayId = explicitSessionReplayIds.find((sessionReplayId) => !explicitSessionReplayIdsFound.has(sessionReplayId));
+      if (missingSessionReplayId != null) {
+        throw new StatusError(StatusError.BadRequest, `Unknown session_replay_id: ${missingSessionReplayId}`);
+      }
+
+      if (auth.type === "client") {
+        const invalidClientSessionReplay = explicitSessionReplays.find((sessionReplay) => sessionReplay.refreshTokenId !== auth.refreshTokenId);
+        if (invalidClientSessionReplay != null) {
+          throw new StatusError(StatusError.BadRequest, "Client analytics events can only reference the current session replay");
+        }
+      }
+    }
+
+    let recentSessionReplayId: string | null = null;
+    if (prisma && auth.refreshTokenId && needsFallbackSessionReplayId) {
+      const recentSession = await findRecentSessionReplay(prisma, {
+        tenancyId: auth.tenancy.id,
+        refreshTokenId: auth.refreshTokenId,
+      });
+      recentSessionReplayId = recentSession?.id ?? null;
+    }
+
+    const rows = body.events.map((event) => {
+      const sessionReplayId = event.session_replay_id ?? body.session_replay_id ?? recentSessionReplayId;
+      const sessionReplaySegmentId = event.session_replay_segment_id ?? body.session_replay_segment_id ?? null;
+      const explicitParentSpanIds = event.parent_span_ids ?? [];
+      // Auto-link events to their segment span (segment_id IS the span_id)
+      const parentSpanIds = sessionReplaySegmentId && !explicitParentSpanIds.includes(sessionReplaySegmentId)
+        ? [sessionReplaySegmentId, ...explicitParentSpanIds]
+        : explicitParentSpanIds;
+
+      return {
+        event_type: event.event_type,
+        event_id: event.event_id || randomUUID(),
+        trace_id: event.trace_id ?? null,
+        event_at: new Date(event.event_at_ms),
+        parent_span_ids: parentSpanIds,
+        data: event.data,
+        project_id: projectId,
+        branch_id: branchId,
+        user_id: auth.type === "client" ? clientUserId : event.user_id ?? defaultUserId,
+        team_id: auth.type === "client" ? null : event.team_id ?? null,
+        refresh_token_id: auth.refreshTokenId ?? null,
+        session_replay_id: sessionReplayId,
+        session_replay_segment_id: sessionReplaySegmentId,
+        from_server: auth.type !== "client",
+      };
     });
 
+    await insertAnalyticsEvents(rows);
+
     return {
       statusCode: 200,
       bodyType: "json",