feat(cli): posthog feature flags + fast_provision experiment

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openrouter/spawn",
-  "version": "1.0.22",
+  "version": "1.0.23",
   "type": "module",
   "bin": {
     "spawn": "cli.js"

packages/cli/src/__tests__/feature-flags.test.ts

@@ -0,0 +1,228 @@
+// Unit tests for shared/feature-flags.ts — fetch, cache, exposure events.
+
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import {
+  _awaitBackgroundRefreshForTest,
+  _resetFeatureFlagsForTest,
+  getFeatureFlag,
+  initFeatureFlags,
+} from "../shared/feature-flags.js";
+import { _resetInstallIdCache } from "../shared/install-id.js";
+import { getSpawnDir } from "../shared/paths.js";
+
+const cachePath = (): string => join(getSpawnDir(), "feature-flags-cache.json");
+
+function writeCache(flags: Record<string, string | boolean>, ageMs = 0): void {
+  const path = cachePath();
+  if (!existsSync(dirname(path))) {
+    mkdirSync(dirname(path), {
+      recursive: true,
+    });
+  }
+  writeFileSync(
+    path,
+    JSON.stringify({
+      fetchedAt: Date.now() - ageMs,
+      flags,
+    }),
+  );
+}
+
+describe("feature flags", () => {
+  const originalFetch = global.fetch;
+  const originalSpawnHome = process.env.SPAWN_HOME;
+  const originalDisabled = process.env.SPAWN_FEATURE_FLAGS_DISABLED;
+  let testHome: string;
+
+  beforeEach(() => {
+    // Pin SPAWN_HOME to a fresh dir under the sandboxed HOME — other tests in
+    // the suite mutate it and don't always restore. We need a known-empty dir
+    // for the cache tests. SPAWN_HOME is required to live inside HOME so we
+    // mkdtemp inside the preload-provided test HOME, not the system tmp.
+    testHome = mkdtempSync(join(process.env.HOME ?? "", "spawn-ff-test-"));
+    process.env.SPAWN_HOME = testHome;
+    _resetFeatureFlagsForTest();
+    _resetInstallIdCache();
+    delete process.env.SPAWN_FEATURE_FLAGS_DISABLED;
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+    if (originalSpawnHome === undefined) {
+      delete process.env.SPAWN_HOME;
+    } else {
+      process.env.SPAWN_HOME = originalSpawnHome;
+    }
+    if (originalDisabled === undefined) {
+      delete process.env.SPAWN_FEATURE_FLAGS_DISABLED;
+    } else {
+      process.env.SPAWN_FEATURE_FLAGS_DISABLED = originalDisabled;
+    }
+    rmSync(testHome, {
+      recursive: true,
+      force: true,
+    });
+  });
+
+  describe("initFeatureFlags", () => {
+    it("populates flags from a successful /decide response", async () => {
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              featureFlags: {
+                fast_provision: "test",
+                other: true,
+              },
+            }),
+          ),
+        ),
+      );
+      await initFeatureFlags();
+      expect(getFeatureFlag("fast_provision", "control")).toBe("test");
+      expect(getFeatureFlag("other", false)).toBe(true);
+    });
+
+    it("falls open on a network error — getFeatureFlag returns the fallback", async () => {
+      global.fetch = mock(() => Promise.reject(new Error("network down")));
+      await initFeatureFlags();
+      expect(getFeatureFlag("fast_provision", "control")).toBe("control");
+    });
+
+    it("falls open on HTTP 500", async () => {
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response("oops", {
+            status: 500,
+          }),
+        ),
+      );
+      await initFeatureFlags();
+      expect(getFeatureFlag("fast_provision", "control")).toBe("control");
+    });
+
+    it("falls open on malformed JSON", async () => {
+      global.fetch = mock(() => Promise.resolve(new Response("not json")));
+      await initFeatureFlags();
+      expect(getFeatureFlag("fast_provision", "control")).toBe("control");
+    });
+
+    it("serves stale cache (>1h old) immediately and refreshes in the background", async () => {
+      writeCache(
+        {
+          fast_provision: "stale",
+        },
+        2 * 60 * 60 * 1000,
+      );
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              featureFlags: {
+                fast_provision: "fresh",
+              },
+            }),
+          ),
+        ),
+      );
+      await initFeatureFlags();
+      // Stale value is served immediately — this is the point of SWR.
+      expect(getFeatureFlag("fast_provision", "control")).toBe("stale");
+      // After the background refresh settles, the fresh value takes over.
+      await _awaitBackgroundRefreshForTest();
+      _resetFeatureFlagsForTest();
+      await initFeatureFlags();
+      expect(getFeatureFlag("fast_provision", "control")).toBe("fresh");
+    });
+
+    it("does NOT fetch when cache is fresh (<1h old)", async () => {
+      writeCache(
+        {
+          fast_provision: "cached",
+        },
+        5 * 60 * 1000,
+      );
+      let fetched = false;
+      global.fetch = mock(() => {
+        fetched = true;
+        return Promise.resolve(new Response("{}"));
+      });
+      await initFeatureFlags();
+      expect(fetched).toBe(false);
+      expect(getFeatureFlag("fast_provision", "control")).toBe("cached");
+    });
+
+    it("writes the response to the cache file", async () => {
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              featureFlags: {
+                fast_provision: "test",
+              },
+            }),
+          ),
+        ),
+      );
+      await initFeatureFlags();
+      expect(existsSync(cachePath())).toBe(true);
+    });
+
+    it("short-circuits when SPAWN_FEATURE_FLAGS_DISABLED=1 is set", async () => {
+      process.env.SPAWN_FEATURE_FLAGS_DISABLED = "1";
+      let fetched = false;
+      global.fetch = mock(() => {
+        fetched = true;
+        return Promise.resolve(new Response("{}"));
+      });
+      await initFeatureFlags();
+      expect(fetched).toBe(false);
+      expect(getFeatureFlag("fast_provision", "control")).toBe("control");
+    });
+
+    it("is idempotent — second call does not refetch", async () => {
+      let count = 0;
+      global.fetch = mock(() => {
+        count++;
+        return Promise.resolve(
+          new Response(
+            JSON.stringify({
+              featureFlags: {
+                fast_provision: "test",
+              },
+            }),
+          ),
+        );
+      });
+      await initFeatureFlags();
+      await initFeatureFlags();
+      expect(count).toBe(1);
+    });
+  });
+
+  describe("getFeatureFlag", () => {
+    it("returns fallback when flags were never initialized", () => {
+      expect(getFeatureFlag("missing", "default")).toBe("default");
+      expect(getFeatureFlag("missing-bool", false)).toBe(false);
+    });
+
+    it("returns fallback for unknown keys when flags are loaded", async () => {
+      global.fetch = mock(() =>
+        Promise.resolve(
+          new Response(
+            JSON.stringify({
+              featureFlags: {
+                known: "yes",
+              },
+            }),
+          ),
+        ),
+      );
+      await initFeatureFlags();
+      expect(getFeatureFlag("known", "default")).toBe("yes");
+      expect(getFeatureFlag("unknown", "default")).toBe("default");
+    });
+  });
+});

packages/cli/src/__tests__/install-id.test.ts

@@ -0,0 +1,48 @@
+// Unit tests for shared/install-id.ts — persistent UUID generation and read.
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { _resetInstallIdCache, getInstallId } from "../shared/install-id.js";
+import { getInstallIdPath } from "../shared/paths.js";
+
+describe("getInstallId", () => {
+  beforeEach(() => {
+    _resetInstallIdCache();
+    const path = getInstallIdPath();
+    if (existsSync(path)) {
+      rmSync(path);
+    }
+  });
+
+  afterEach(() => {
+    _resetInstallIdCache();
+  });
+
+  it("creates a UUID on first call and persists it", () => {
+    const id = getInstallId();
+    expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/);
+    expect(existsSync(getInstallIdPath())).toBe(true);
+    expect(readFileSync(getInstallIdPath(), "utf8").trim()).toBe(id);
+  });
+
+  it("returns the same value on subsequent calls (in-memory cache)", () => {
+    const a = getInstallId();
+    const b = getInstallId();
+    expect(a).toBe(b);
+  });
+
+  it("reads from disk on a fresh module state", () => {
+    const a = getInstallId();
+    _resetInstallIdCache();
+    const b = getInstallId();
+    expect(a).toBe(b);
+  });
+
+  it("regenerates if the persisted file is malformed", () => {
+    writeFileSync(getInstallIdPath(), "not-a-uuid");
+    _resetInstallIdCache();
+    const id = getInstallId();
+    expect(id).toMatch(/^[0-9a-f-]{36}$/);
+    expect(id).not.toBe("not-a-uuid");
+  });
+});

packages/cli/src/index.ts

@@ -39,6 +39,7 @@ import {
 } from "./commands/index.js";
 import { expandEqualsFlags, findUnknownFlag } from "./flags.js";
 import { agentKeys, cloudKeys, getCacheAge, loadManifest } from "./manifest.js";
+import { getFeatureFlag, initFeatureFlags } from "./shared/feature-flags.js";
 import { getInstallRefPath } from "./shared/paths.js";
 import { asyncTryCatch, asyncTryCatchIf, isFileError, isNetworkError, tryCatch, tryCatchIf } from "./shared/result.js";
 import { captureError, initTelemetry, setTelemetryContext } from "./shared/telemetry.js";
@@ -848,6 +849,8 @@ async function main(): Promise<void> {
   // ── `spawn pick` — bypass all flag parsing; used by bash scripts ──────────
   // Must be handled before expandEqualsFlags / resolvePrompt so that pick's
   // own --prompt flag is not mistakenly consumed by the top-level prompt logic.
+  // Runs before initFeatureFlags() — this is a hot path called by shell
+  // scripts and must stay fast; it has no code paths that gate on a flag.
   if (rawArgs[0] === "pick") {
     const pickResult = await asyncTryCatch(() => cmdPick(expandEqualsFlags(rawArgs.slice(1))));
     if (!pickResult.ok) {
@@ -857,11 +860,18 @@ async function main(): Promise<void> {
   }
 
   // ── `spawn feedback` — bypass flag parsing; rest of args are the message ───
+  // Also runs before initFeatureFlags() for the same reason as `pick`.
   if (rawArgs[0] === "feedback") {
     await cmdFeedback(rawArgs.slice(1));
     return;
   }
 
+  // Fetch feature flags (1.5s timeout, fail-open). Must run before any code
+  // path that gates on a flag — currently the SPAWN_BETA composition for the
+  // `fast_provision` experiment. Placed AFTER the pick/feedback bypasses so
+  // those fast paths never pay the flag-fetch cost.
+  await initFeatureFlags();
+
   const args = expandEqualsFlags(rawArgs);
 
   // Pre-scan for --output json before checkForUpdates() so install script
@@ -927,6 +937,7 @@ async function main(): Promise<void> {
     "skills",
   ]);
   const betaFeatures = extractAllFlagValues(filteredArgs, "--beta", "spawn <agent> <cloud> --beta parallel");
+  const userOptedIntoBeta = betaFeatures.length > 0 || process.env.SPAWN_FAST === "1";
   for (const flag of betaFeatures) {
     if (!VALID_BETA_FEATURES.has(flag)) {
       console.error(pc.red(`Unknown beta feature: ${pc.bold(flag)}`));
@@ -945,6 +956,18 @@ async function main(): Promise<void> {
   if (process.env.SPAWN_FAST === "1") {
     betaFeatures.push("tarball", "images", "parallel", "docker");
   }
+
+  // fast_provision experiment: if the user did NOT pass --beta or --fast,
+  // bucket them on the PostHog `fast_provision` flag. The `test` variant
+  // turns on tarball + images by default; control behaves as before.
+  // Exposure is captured for both variants so PostHog can compute conversion.
+  if (!userOptedIntoBeta) {
+    const variant = getFeatureFlag("fast_provision", "control");
+    if (variant === "test") {
+      betaFeatures.push("tarball", "images");
+    }
+  }
+
   if (betaFeatures.length > 0) {
     process.env.SPAWN_BETA = [
       ...new Set(betaFeatures),