From a1da657dd990a04b7c8452fff433aa75df001896 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Tue, 17 Feb 2026 14:07:11 -0800 Subject: [PATCH 01/21] fix replay pausing issue --- .../analytics/replays/session-replay-machine.test.ts | 8 ++++++-- .../analytics/replays/session-replay-machine.ts | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.test.ts b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.test.ts index 889b7ec320..5b748078a0 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.test.ts +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.test.ts @@ -557,16 +557,18 @@ describe("session-replay-machine", () => { describe("TOGGLE_PLAY_PAUSE", () => { it("pauses from playing", () => { - const state = twoTabReadyState({ playbackMode: "playing" }); + const state = twoTabReadyState({ playbackMode: "playing", currentGlobalTimeMsForUi: 2500 }); const { state: s, effects } = dispatch(state, { type: "TOGGLE_PLAY_PAUSE", nowMs: 1000 }); expect(s.playbackMode).toBe("paused"); + expect(s.pausedAtGlobalMs).toBe(2500); expect(hasEffect(effects, "pause_all")).toBe(true); }); it("pauses from gap_fast_forward", () => { - const state = twoTabReadyState({ playbackMode: "gap_fast_forward" }); + const state = twoTabReadyState({ playbackMode: "gap_fast_forward", currentGlobalTimeMsForUi: 3000 }); const { state: s } = dispatch(state, { type: "TOGGLE_PLAY_PAUSE", nowMs: 1000 }); expect(s.playbackMode).toBe("paused"); + expect(s.pausedAtGlobalMs).toBe(3000); expect(s.gapFastForward).toBeNull(); }); @@ -575,9 +577,11 @@ describe("session-replay-machine", () => { playbackMode: "buffering", bufferingAtGlobalMs: 1000, autoResumeAfterBuffering: true, + currentGlobalTimeMsForUi: 1500, }); const { state: s } = dispatch(state, { type: "TOGGLE_PLAY_PAUSE", nowMs: 1000 }); expect(s.playbackMode).toBe("paused"); + expect(s.pausedAtGlobalMs).toBe(1500); expect(s.bufferingAtGlobalMs).toBeNull(); expect(s.autoResumeAfterBuffering).toBe(false); }); diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.ts b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.ts index 2d3ec80809..30cbff6525 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.ts +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/analytics/replays/session-replay-machine.ts @@ -761,6 +761,7 @@ export function replayReducer(state: ReplayState, action: ReplayAction): Reducer state: { ...state, playbackMode: "paused", + pausedAtGlobalMs: state.currentGlobalTimeMsForUi, gapFastForward: null, bufferingAtGlobalMs: null, autoResumeAfterBuffering: false, From 39ca118771d2611dfea4c43173942143fff57534 Mon Sep 17 00:00:00 2001 From: Bilal Godil Date: Tue, 17 Feb 2026 15:20:24 -0800 Subject: [PATCH 02/21] analytics event tracking --- apps/backend/scripts/clickhouse-migrations.ts | 2 + .../latest/analytics/events/batch/route.tsx | 102 ++++ .../latest/session-replays/batch/route.tsx | 20 +- apps/backend/src/lib/session-replays.tsx | 23 + .../api/v1/analytics-events-batch.test.ts | 435 ++++++++++++++++++ .../src/interface/client-interface.ts | 22 + .../apps/implementations/client-app-impl.ts | 21 + .../apps/implementations/event-tracker.ts | 279 +++++++++++ .../apps/implementations/session-replay.ts | 10 +- .../stack-app/apps/interfaces/client-app.ts | 1 + 10 files changed, 892 insertions(+), 23 deletions(-) create mode 100644 apps/backend/src/app/api/latest/analytics/events/batch/route.tsx create mode 100644 apps/backend/src/lib/session-replays.tsx create mode 100644 apps/e2e/tests/backend/endpoints/api/v1/analytics-events-batch.test.ts create mode 100644 packages/template/src/lib/stack-app/apps/implementations/event-tracker.ts diff --git a/apps/backend/scripts/clickhouse-migrations.ts b/apps/backend/scripts/clickhouse-migrations.ts index 8f3f15c5a0..d94f84baa0 100644 --- a/apps/backend/scripts/clickhouse-migrations.ts +++ b/apps/backend/scripts/clickhouse-migrations.ts @@ -20,6 +20,8 @@ export async function runClickhouseMigrations() { await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL }); await client.exec({ query: BACKFILL_REFRESH_TOKEN_ID_COLUMN_SQL }); await client.exec({ query: SIGN_UP_RULE_TRIGGER_EVENT_ROW_FORMAT_MUTATION_SQL }); + // Recreate the events view so SELECT * picks up columns added by EVENTS_ADD_REPLAY_COLUMNS_SQL + await client.exec({ query: EVENTS_VIEW_SQL }); const queries = [ "REVOKE ALL PRIVILEGES ON *.* FROM limited_user;", "REVOKE ALL FROM limited_user;", diff --git a/apps/backend/src/app/api/latest/analytics/events/batch/route.tsx b/apps/backend/src/app/api/latest/analytics/events/batch/route.tsx new file mode 100644 index 0000000000..bc4e2df581 --- /dev/null +++ b/apps/backend/src/app/api/latest/analytics/events/batch/route.tsx @@ -0,0 +1,102 @@ +import { getClickhouseAdminClient } from "@/lib/clickhouse"; +import { findRecentSessionReplay } from "@/lib/session-replays"; +import { getPrismaClientForTenancy } from "@/prisma-client"; +import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; +import { KnownErrors } from "@stackframe/stack-shared"; +import { adaptSchema, clientOrHigherAuthTypeSchema, yupArray, yupMixed, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; +import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const MAX_EVENTS = 500; + +export const POST = createSmartRouteHandler({ + metadata: { + summary: "Upload analytics event batch", + description: "Uploads a batch of auto-captured analytics events ($page-view, $click).", + tags: ["Analytics Events"], + hidden: true, + }, + request: yupObject({ + auth: yupObject({ + type: clientOrHigherAuthTypeSchema, + tenancy: adaptSchema, + user: adaptSchema, + refreshTokenId: adaptSchema, + }).defined(), + body: yupObject({ + session_replay_segment_id: yupString().defined().matches(UUID_RE, "Invalid session_replay_segment_id"), + batch_id: yupString().defined().matches(UUID_RE, "Invalid batch_id"), + sent_at_ms: yupNumber().defined().integer().min(0), + events: yupArray( + yupObject({ + event_type: yupString().defined().oneOf(["$page-view", "$click"]), + event_at_ms: yupNumber().defined().integer().min(0), + data: yupMixed().defined(), + }).defined(), + ).defined().min(1).max(MAX_EVENTS), + }).defined(), + }), + response: yupObject({ + statusCode: yupNumber().oneOf([200]).defined(), + bodyType: yupString().oneOf(["json"]).defined(), + body: yupObject({ + inserted: yupNumber().defined(), + }).defined(), + }), + async handler({ auth, body }) { + if (!auth.tenancy.config.apps.installed["analytics"]?.enabled) { + return { + statusCode: 200, + bodyType: "json", + body: { inserted: 0 }, + }; + } + if (!auth.user) { + throw new KnownErrors.UserAuthenticationRequired(); + } + if (!auth.refreshTokenId) { + throw new StatusError(StatusError.BadRequest, "A refresh token is required for analytics events"); + } + + const projectId = auth.tenancy.project.id; + const branchId = auth.tenancy.branchId; + const userId = auth.user.id; + const refreshTokenId = auth.refreshTokenId; + const tenancyId = auth.tenancy.id; + + const prisma = await getPrismaClientForTenancy(auth.tenancy); + const recentSession = await findRecentSessionReplay(prisma, { tenancyId, refreshTokenId }); + + const clickhouseClient = getClickhouseAdminClient(); + + const rows = body.events.map((event) => ({ + event_type: event.event_type, + event_at: new Date(event.event_at_ms), + data: event.data as Record, + project_id: projectId, + branch_id: branchId, + user_id: userId, + team_id: null, + refresh_token_id: refreshTokenId, + session_replay_id: recentSession?.id ?? null, + session_replay_segment_id: body.session_replay_segment_id, + })); + + await clickhouseClient.insert({ + table: "analytics_internal.events", + values: rows, + format: "JSONEachRow", + clickhouse_settings: { + date_time_input_format: "best_effort", + async_insert: 1, + }, + }); + + return { + statusCode: 200, + bodyType: "json", + body: { inserted: body.events.length }, + }; + }, +}); diff --git a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx index 3ec4c6d0cf..e4af4323b8 100644 --- a/apps/backend/src/app/api/latest/session-replays/batch/route.tsx +++ b/apps/backend/src/app/api/latest/session-replays/batch/route.tsx @@ -2,6 +2,7 @@ import { getPrismaClientForTenancy } from "@/prisma-client"; import { uploadBytes } from "@/s3"; import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler"; import { Prisma } from "@/generated/prisma/client"; +import { findRecentSessionReplay } from "@/lib/session-replays"; import { KnownErrors } from "@stackframe/stack-shared"; import { adaptSchema, clientOrHigherAuthTypeSchema, yupArray, yupMixed, yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { StatusError } from "@stackframe/stack-shared/dist/utils/errors"; @@ -15,8 +16,6 @@ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0 const MAX_BODY_BYTES = 5_000_000; const MAX_EVENTS = 5_000; -const SESSION_IDLE_TIMEOUT_MS = 3 * 60 * 1000; -const MAX_SESSION_DURATION_MS = 12 * 60 * 60 * 1000; function extractEventTimesMs(events: unknown[], fallbackMs: number) { let minTs = Infinity; @@ -114,22 +113,7 @@ export const POST = createSmartRouteHandler({ const { firstMs, lastMs } = extractEventTimesMs(body.events, body.sent_at_ms); const prisma = await getPrismaClientForTenancy(auth.tenancy); - - // Find a recent session replay for this refresh token (temporal grouping). - // If the last batch arrived within SESSION_IDLE_TIMEOUT_MS, reuse that replay. - // Also enforce a max session duration so replays don't grow indefinitely. - const cutoff = new Date(Date.now() - SESSION_IDLE_TIMEOUT_MS); - const maxDurationCutoff = new Date(Date.now() - MAX_SESSION_DURATION_MS); - const recentSession = await prisma.sessionReplay.findFirst({ - where: { - tenancyId, - refreshTokenId, - updatedAt: { gte: cutoff }, - startedAt: { gte: maxDurationCutoff }, - }, - orderBy: { updatedAt: "desc" }, - select: { id: true, startedAt: true, lastEventAt: true }, - }); + const recentSession = await findRecentSessionReplay(prisma, { tenancyId, refreshTokenId }); const replayId = recentSession?.id ?? randomUUID(); const s3Key = `session-replays/${projectId}/${branchId}/${replayId}/${batchId}.json.gz`; diff --git a/apps/backend/src/lib/session-replays.tsx b/apps/backend/src/lib/session-replays.tsx new file mode 100644 index 0000000000..e6b653b446 --- /dev/null +++ b/apps/backend/src/lib/session-replays.tsx @@ -0,0 +1,23 @@ +import { PrismaClient } from "@/generated/prisma/client"; +import { PrismaClientWithReplica } from "@/prisma-client"; + +export const SESSION_IDLE_TIMEOUT_MS = 3 * 60 * 1000; +export const MAX_SESSION_DURATION_MS = 12 * 60 * 60 * 1000; + +export async function findRecentSessionReplay(prisma: PrismaClientWithReplica, options: { + tenancyId: string, + refreshTokenId: string, +}) { + const cutoff = new Date(Date.now() - SESSION_IDLE_TIMEOUT_MS); + const maxDurationCutoff = new Date(Date.now() - MAX_SESSION_DURATION_MS); + return await prisma.sessionReplay.findFirst({ + where: { + tenancyId: options.tenancyId, + refreshTokenId: options.refreshTokenId, + updatedAt: { gte: cutoff }, + startedAt: { gte: maxDurationCutoff }, + }, + orderBy: { updatedAt: "desc" }, + select: { id: true, startedAt: true, lastEventAt: true }, + }); +} diff --git a/apps/e2e/tests/backend/endpoints/api/v1/analytics-events-batch.test.ts b/apps/e2e/tests/backend/endpoints/api/v1/analytics-events-batch.test.ts new file mode 100644 index 0000000000..7ec66f7b6a --- /dev/null +++ b/apps/e2e/tests/backend/endpoints/api/v1/analytics-events-batch.test.ts @@ -0,0 +1,435 @@ +import { randomUUID } from "node:crypto"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; +import { it } from "../../../../helpers"; +import { Auth, Project, backendContext, niceBackendFetch } from "../../../backend-helpers"; + +async function uploadEventBatch(options: { + sessionReplaySegmentId: string, + batchId: string, + sentAtMs: number, + events: { event_type: string, event_at_ms: number, data: unknown }[], +}) { + return await niceBackendFetch("/api/v1/analytics/events/batch", { + method: "POST", + accessType: "client", + body: { + session_replay_segment_id: options.sessionReplaySegmentId, + batch_id: options.batchId, + sent_at_ms: options.sentAtMs, + events: options.events, + }, + }); +} + +it("requires a user token", async ({ expect }) => { + await Project.createAndSwitch({ config: { magic_link_enabled: true } }); + await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } }); + backendContext.set({ userAuth: null }); + + const res = await niceBackendFetch("/api/v1/analytics/events/batch", { + method: "POST", + accessType: "client", + body: { + session_replay_segment_id: randomUUID(), + batch_id: randomUUID(), + sent_at_ms: Date.now(), + events: [{ event_type: "$page-view", event_at_ms: Date.now(), data: {} }], + }, + }); + + expect(res).toMatchInlineSnapshot(` + NiceResponse { + "status": 401, + "body": { + "code": "USER_AUTHENTICATION_REQUIRED", + "error": "User authentication required for this endpoint.", + }, + "headers": Headers { + "x-stack-known-error": "USER_AUTHENTICATION_REQUIRED", +