From 05f9921c830c4df2c54569e77c59f5fa1bedbee0 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 7 Aug 2026 02:41:04 -0700 Subject: [PATCH] feat(server): record runtime mode per turn and on mode changes Runtime mode was only reported on provider.session.started, which overcounts users who toggle modes often since every toggle restarts the session, and cannot be crossed with interactionMode. Mode switches were not reported at all. Adds runtimeMode to the provider.turn.sent payload for a usage-weighted view, and a provider.runtime_mode.changed event carrying from/to so the supervised-to-full-access funnel is answerable. Co-Authored-By: Claude Opus 5 (1M context) --- .../providerService.integration.test.ts | 138 ++++++++++++++---- .../src/provider/Layers/ProviderService.ts | 20 +++ 2 files changed, 129 insertions(+), 29 deletions(-) diff --git a/apps/server/integration/providerService.integration.test.ts b/apps/server/integration/providerService.integration.test.ts index e703af4b1f4..c57d289f992 100644 --- a/apps/server/integration/providerService.integration.test.ts +++ b/apps/server/integration/providerService.integration.test.ts @@ -8,6 +8,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import { ProviderAdapterRegistry } from "../src/provider/Services/ProviderAdapterRegistry.ts"; @@ -54,35 +55,58 @@ interface IntegrationFixture { readonly layer: Layer.Layer; } -const makeIntegrationFixture = Effect.gen(function* () { - const cwd = yield* makeWorkspaceDirectory; - const harness = yield* makeTestProviderAdapterHarness(); - - const registry = makeAdapterRegistryMock({ - [ProviderDriverKind.make("codex")]: harness.adapter, - }); +interface RecordedAnalyticsEvent { + readonly event: string; + readonly properties: Readonly> | undefined; +} - const directoryLayer = ProviderSessionDirectoryLive.pipe( - Layer.provide(ProviderSessionRuntime.layer), +/** + * Analytics layer that keeps captured events in memory so tests can assert on + * telemetry payloads. `AnalyticsService.layerTest` discards them. + */ +const makeRecordingAnalytics = Effect.gen(function* () { + const recorded = yield* Ref.make>([]); + const layer = Layer.succeed( + AnalyticsService, + AnalyticsService.of({ + record: (event, properties) => + Ref.update(recorded, (current) => [...current, { event, properties }]), + flush: Effect.void, + }), ); - - const shared = Layer.mergeAll( - directoryLayer, - Layer.succeed(ProviderAdapterRegistry, registry), - ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS), - AnalyticsService.layerTest, - Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers), - ).pipe(Layer.provide(SqlitePersistenceMemory)); - - const layer = makeProviderServiceLive().pipe(Layer.provide(shared)); - - return { - cwd, - harness, - layer, - } satisfies IntegrationFixture; + return { layer, get: Ref.get(recorded) } as const; }); +const makeIntegrationFixture = (options?: { readonly analytics?: Layer.Layer }) => + Effect.gen(function* () { + const cwd = yield* makeWorkspaceDirectory; + const harness = yield* makeTestProviderAdapterHarness(); + + const registry = makeAdapterRegistryMock({ + [ProviderDriverKind.make("codex")]: harness.adapter, + }); + + const directoryLayer = ProviderSessionDirectoryLive.pipe( + Layer.provide(ProviderSessionRuntime.layer), + ); + + const shared = Layer.mergeAll( + directoryLayer, + Layer.succeed(ProviderAdapterRegistry, registry), + ServerSettingsService.layerTest(DEFAULT_SERVER_SETTINGS), + options?.analytics ?? AnalyticsService.layerTest, + Layer.succeed(ProviderEventLoggers, NoOpProviderEventLoggers), + ).pipe(Layer.provide(SqlitePersistenceMemory)); + + const layer = makeProviderServiceLive().pipe(Layer.provide(shared)); + + return { + cwd, + harness, + layer, + } satisfies IntegrationFixture; + }); + const collectEventsDuring = ( stream: Stream.Stream, count: number, @@ -126,7 +150,7 @@ const runTurn = (input: { it.live("replays typed runtime fixture events", () => Effect.gen(function* () { - const fixture = yield* makeIntegrationFixture; + const fixture = yield* makeIntegrationFixture(); yield* Effect.gen(function* () { const provider = yield* ProviderService; @@ -161,7 +185,7 @@ it.live("replays typed runtime fixture events", () => it.live("replays file-changing fixture turn events", () => Effect.gen(function* () { - const fixture = yield* makeIntegrationFixture; + const fixture = yield* makeIntegrationFixture(); const { join } = yield* Path.Path; const { writeFileString } = yield* FileSystem.FileSystem; @@ -198,7 +222,7 @@ it.live("replays file-changing fixture turn events", () => it.live("runs multi-turn tool/approval flow", () => Effect.gen(function* () { - const fixture = yield* makeIntegrationFixture; + const fixture = yield* makeIntegrationFixture(); const { join } = yield* Path.Path; const { writeFileString } = yield* FileSystem.FileSystem; @@ -250,7 +274,7 @@ it.live("runs multi-turn tool/approval flow", () => it.live("rolls back provider conversation state only", () => Effect.gen(function* () { - const fixture = yield* makeIntegrationFixture; + const fixture = yield* makeIntegrationFixture(); const { join } = yield* Path.Path; const { writeFileString, readFileString } = yield* FileSystem.FileSystem; @@ -302,3 +326,59 @@ it.live("rolls back provider conversation state only", () => }).pipe(Effect.provide(fixture.layer)); }).pipe(Effect.provide(NodeServices.layer)), ); + +it.live("reports runtime mode per turn and on mode transitions", () => + Effect.gen(function* () { + const analytics = yield* makeRecordingAnalytics; + const fixture = yield* makeIntegrationFixture({ analytics: analytics.layer }); + const threadId = ThreadId.make("thread-integration-runtime-mode"); + + yield* Effect.gen(function* () { + const provider = yield* ProviderService; + const startSession = (runtimeMode: "approval-required" | "full-access") => + provider.startSession(threadId, { + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: codexInstanceId, + cwd: fixture.cwd, + runtimeMode, + }); + + yield* startSession("approval-required"); + yield* runTurn({ + provider, + harness: fixture.harness, + threadId, + userText: "supervised turn", + response: { events: codexTurnTextFixture }, + }); + + // Toggling the mode restarts the session, which is the only place the + // transition is observable. + yield* startSession("full-access"); + yield* runTurn({ + provider, + harness: fixture.harness, + threadId, + userText: "full access turn", + response: { events: codexTurnTextFixture }, + }); + + const recorded = yield* analytics.get; + + assert.deepEqual( + recorded + .filter((entry) => entry.event === "provider.turn.sent") + .map((entry) => entry.properties?.runtimeMode), + ["approval-required", "full-access"], + ); + + assert.deepEqual( + recorded + .filter((entry) => entry.event === "provider.runtime_mode.changed") + .map((entry) => [entry.properties?.from, entry.properties?.to]), + [["approval-required", "full-access"]], + ); + }).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index ecf26a914c1..d0acc1039c3 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -459,6 +459,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( adapter, instanceId, threadId: input.threadId, + runtimeMode: binding.runtimeMode, isActive: true, } as const; } @@ -468,6 +469,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( adapter, instanceId, threadId: input.threadId, + runtimeMode: binding.runtimeMode, isActive: false, } as const; } @@ -480,6 +482,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( adapter: recovered.adapter, instanceId, threadId: input.threadId, + runtimeMode: recovered.session.runtimeMode, isActive: true, } as const; }); @@ -629,6 +632,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( input.modelSelection.model.trim().length > 0, }); + // Changing runtime mode restarts the session, so the transition is only + // observable here, by diffing against the mode the previous session for + // this thread was bound to. Recording it separately is what makes the + // "started supervised, switched to full access" funnel answerable. + const previousRuntimeMode = persistedBinding?.runtimeMode; + if (previousRuntimeMode !== undefined && previousRuntimeMode !== input.runtimeMode) { + yield* analytics.record("provider.runtime_mode.changed", { + provider: sessionWithInstance.provider, + from: previousRuntimeMode, + to: input.runtimeMode, + }); + } + return sessionWithInstance; }).pipe( withMetrics({ @@ -703,6 +719,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( provider: routed.adapter.provider, model: input.modelSelection?.model, interactionMode: input.interactionMode, + // Session-start events alone skew runtime mode toward users who toggle + // often, since every toggle restarts the session. Recording it per turn + // gives a usage-weighted view and lets it cross with interactionMode. + runtimeMode: routed.runtimeMode, attachmentCount: input.attachments.length, hasInput: typeof input.input === "string" && input.input.trim().length > 0, });