Expand PostHog telemetry to close audited coverage gaps

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## 0.0.11-beta.2 — 2026-05-21
 
+### Features
+- Expand PostHog telemetry coverage to close the 16 server-side and 12 web-UI gaps surfaced by the May audit (#376). Server-side adds `cli_install_success` / `cli_install_failure` / `cli_uninstall_success` / `cli_uninstall_failure` / `cli_list_invoked` / `cli_parse_error` / `cli_unexpected_error` / `hook_dispatch_error` (CLI lifecycle outcomes in `bin/failproofai.mjs`), `hook_stdin_error` / `hook_payload_parse_error` (hook handler input errors in `src/hooks/handler.ts`), `policy_evaluation_error` (builtin policy crashes in `src/hooks/policy-evaluator.ts`, distinct from the existing `custom_hook_error`), `custom_policy_validation_failed` / `custom_hooks_load_error` / `policy_params_validation_warning` / `scope_validation_failed` / `hook_write_failed` / `multi_scope_warning_shown` / `cli_detection_summary` / `beta_policies_installed` (manager / loader / install-prompt internals), and `first_install` / `version_changed` (lifecycle detection in `scripts/postinstall.mjs` via a new `~/.failproofai/last-version` file). Web-UI adds `policies_tab_switched` / `activity_filter_changed` (debounced) / `activity_row_toggled` / `activity_copy_clicked` / `activity_pagination_changed` / `cli_selection_toggled` / `cli_install_remove_submitted` / `cli_reinstall_submitted` / `policy_config_modal_opened` / `policy_config_modal_closed` / `action_error_displayed` / `hooks_install_from_error_clicked` via `usePostHog()` in `app/policies/hooks-client.tsx`. The deny-/instruct-only condition at `handler.ts:344` (allow-path tracking) is intentionally left unchanged. All events go through the existing helpers (`trackHookEvent`, `trackInstallEvent`, `captureClientEvent`) and honor `FAILPROOFAI_TELEMETRY_DISABLED=1`.
+
 ### Breaking
 - Remove the undocumented cloud auth + event relay subsystem ahead of a from-scratch redesign. Deletes `src/auth/` (OAuth 2.0 device-flow login against `api.befailproof.ai`, `~/.failproofai/auth.json` token store) and `src/relay/` (WebSocket event relay daemon, sanitized JSONL queue at `~/.failproofai/cache/server-queue/`, PID tracking). Strips the `failproofai login` / `logout` / `whoami` / `relay start|stop|status` / `sync` subcommands and the internal `--relay-daemon` mode from `bin/failproofai.mjs`, along with their `--help` entries and "did you mean" suggestions. Removes the fire-and-forget `appendToServerQueue` + `ensureRelayRunning` calls from `src/hooks/handler.ts` so hook evaluation no longer enqueues events or lazy-spawns a daemon. The whole subsystem had zero references in `README.md`, `docs/`, `examples/`, or `__tests__/`, and only had internal cross-imports — `tsc`, `eslint`, `vitest` (1623 tests), and the `bun run build` bundles all stay green. Users who ran `failproofai login` should also wipe `~/.failproofai/{auth.json,cache/server-queue,relay.pid}` and stop any running relay daemon by hand; new auth/cloud surface will land in a follow-up.
 

__tests__/hooks/new-telemetry.test.ts

@@ -0,0 +1,211 @@
+// @vitest-environment node
+/**
+ * Coverage for telemetry events added as part of the gap-closing audit.
+ * Each test stubs trackHookEvent and asserts the right event name + props
+ * fire at the trigger site. Keep one focused case per event.
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { resolve } from "node:path";
+import { homedir } from "node:os";
+
+vi.mock("node:fs", () => ({
+  readFileSync: vi.fn(),
+  writeFileSync: vi.fn(),
+  existsSync: vi.fn(),
+  mkdirSync: vi.fn(),
+  readdirSync: vi.fn(() => []),
+}));
+
+vi.mock("node:child_process", () => ({
+  execSync: vi.fn(),
+}));
+
+vi.mock("../../src/hooks/install-prompt", async () => {
+  const actual = await vi.importActual<typeof import("../../src/hooks/install-prompt")>(
+    "../../src/hooks/install-prompt",
+  );
+  return {
+    ...actual,
+    promptPolicySelection: vi.fn(() => Promise.resolve(["block-sudo"])),
+  };
+});
+
+vi.mock("../../src/hooks/integrations", () => ({
+  detectInstalledClis: vi.fn(() => ["claude"]),
+  getIntegration: vi.fn((id: string) => ({
+    displayName: id,
+    scopes: id === "codex" ? ["user", "project"] : ["user", "project", "local"],
+    eventTypes: [],
+    getSettingsPath: vi.fn(),
+    hooksInstalledInSettings: vi.fn(),
+    readSettings: vi.fn(() => ({})),
+    writeSettings: vi.fn(),
+    writeHookEntries: vi.fn(),
+    removeHooksFromFile: vi.fn(() => 0),
+  })),
+  claudeCode: {
+    getSettingsPath: vi.fn(() => "/tmp/.claude/settings.json"),
+    hooksInstalledInSettings: vi.fn(() => false),
+  },
+  listIntegrations: vi.fn(() => []),
+}));
+
+vi.mock("../../src/hooks/hooks-config", () => ({
+  readHooksConfig: vi.fn(() => ({ enabledPolicies: [] })),
+  readMergedHooksConfig: vi.fn(() => ({ enabledPolicies: [] })),
+  writeHooksConfig: vi.fn(),
+  readScopedHooksConfig: vi.fn(() => ({ enabledPolicies: [] })),
+  writeScopedHooksConfig: vi.fn(),
+  findProjectConfigDir: vi.fn((cwd: string) => cwd),
+  getConfigPathForScope: vi.fn(() => "/tmp/policies-config.json"),
+}));
+
+vi.mock("../../src/hooks/hook-telemetry", () => ({
+  trackHookEvent: vi.fn(() => Promise.resolve()),
+}));
+
+vi.mock("../../lib/telemetry-id", () => ({
+  getInstanceId: vi.fn(() => "test-instance-id"),
+  hashToId: vi.fn((raw: string) => `hashed:${raw}`),
+}));
+
+vi.mock("../../src/hooks/custom-hooks-loader", async () => {
+  const actual = await vi.importActual<typeof import("../../src/hooks/custom-hooks-loader")>(
+    "../../src/hooks/custom-hooks-loader",
+  );
+  return {
+    ...actual,
+    loadCustomHooks: vi.fn(() => Promise.resolve([])),
+    discoverPolicyFiles: vi.fn(() => []),
+  };
+});
+
+describe("new telemetry events — manager", () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(execSync).mockReturnValue("/usr/local/bin/failproofai\n");
+    vi.spyOn(console, "log").mockImplementation(() => {});
+    vi.spyOn(console, "error").mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("fires scope_validation_failed when scope is unsupported for the CLI", async () => {
+    vi.mocked(existsSync).mockReturnValue(true);
+    vi.mocked(readFileSync).mockReturnValue("{}");
+
+    const { installHooks } = await import("../../src/hooks/manager");
+    const { trackHookEvent } = await import("../../src/hooks/hook-telemetry");
+
+    // codex does not support the "local" scope (see src/hooks/integrations.ts)
+    await expect(installHooks(["block-sudo"], "local" as never, undefined, false, undefined, undefined, false, [
+      "codex",
+    ])).rejects.toThrow(/Scope "local" is not supported/);
+
+    expect(trackHookEvent).toHaveBeenCalledWith(
+      "test-instance-id",
+      "scope_validation_failed",
+      expect.objectContaining({
+        cli: "codex",
+        scope: "local",
+        supported_scopes: expect.any(Array),
+      }),
+    );
+  });
+
+  it("fires custom_policy_validation_failed when the custom file throws", async () => {
+    vi.mocked(existsSync).mockReturnValue(true);
+    vi.mocked(readFileSync).mockReturnValue("{}");
+    const { loadCustomHooks } = await import("../../src/hooks/custom-hooks-loader");
+    vi.mocked(loadCustomHooks).mockRejectedValueOnce(new Error("Custom hooks file not found: /tmp/missing.js"));
+    const exitSpy = vi.spyOn(process, "exit").mockImplementation(((_code?: string | number | null | undefined) => undefined as never) as never);
+
+    const { installHooks } = await import("../../src/hooks/manager");
+    const { trackHookEvent } = await import("../../src/hooks/hook-telemetry");
+
+    await installHooks(["block-sudo"], "user", undefined, false, undefined, "/tmp/missing.js");
+
+    expect(trackHookEvent).toHaveBeenCalledWith(
+      "test-instance-id",
+      "custom_policy_validation_failed",
+      expect.objectContaining({
+        scope: "user",
+        error_type: "file_not_found",
+      }),
+    );
+    exitSpy.mockRestore();
+  });
+
+  it("fires policy_params_validation_warning when an unknown key is in policyParams", async () => {
+    vi.mocked(existsSync).mockReturnValue(false);
+    const { readMergedHooksConfig } = await import("../../src/hooks/hooks-config");
+    vi.mocked(readMergedHooksConfig).mockReturnValue({
+      enabledPolicies: [],
+      policyParams: { "nonexistent-policy": { hint: "test" } },
+    });
+
+    const { listHooks } = await import("../../src/hooks/manager");
+    const { trackHookEvent } = await import("../../src/hooks/hook-telemetry");
+
+    await listHooks();
+
+    expect(trackHookEvent).toHaveBeenCalledWith(
+      "test-instance-id",
+      "policy_params_validation_warning",
+      expect.objectContaining({
+        unknown_keys_count: 1,
+        unknown_keys: ["nonexistent-policy"],
+      }),
+    );
+  });
+
+  it("respects FAILPROOFAI_TELEMETRY_DISABLED — the underlying helper is mocked, but the call still happens (test verifies the trigger fires unconditionally)", async () => {
+    // The helper itself short-circuits when FAILPROOFAI_TELEMETRY_DISABLED=1
+    // (verified in __tests__/lib/telemetry.test.ts). Here we just confirm the
+    // trigger call site still runs — the env-var check lives in the helper.
+    process.env.FAILPROOFAI_TELEMETRY_DISABLED = "1";
+    vi.mocked(existsSync).mockReturnValue(true);
+    vi.mocked(readFileSync).mockReturnValue("{}");
+
+    try {
+      const { installHooks } = await import("../../src/hooks/manager");
+      const { trackHookEvent } = await import("../../src/hooks/hook-telemetry");
+
+      await installHooks(["block-sudo"], "user");
+
+      // The call site fires; the helper internally no-ops. This is the same
+      // contract as the existing hooks_installed test.
+      expect(trackHookEvent).toHaveBeenCalled();
+    } finally {
+      delete process.env.FAILPROOFAI_TELEMETRY_DISABLED;
+    }
+  });
+});
+
+describe("new telemetry events — install-prompt", () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it("fires cli_detection_summary with resolution_mode=explicit when --cli is passed", async () => {
+    const { resolveTargetClis } = await import("../../src/hooks/install-prompt");
+    const { trackHookEvent } = await import("../../src/hooks/hook-telemetry");
+
+    const result = await resolveTargetClis(["codex"], "install");
+    expect(result).toEqual(["codex"]);
+    expect(trackHookEvent).toHaveBeenCalledWith(
+      "test-instance-id",
+      "cli_detection_summary",
+      expect.objectContaining({
+        action: "install",
+        explicit_clis: ["codex"],
+        selected_clis: ["codex"],
+        resolution_mode: "explicit",
+      }),
+    );
+  });
+});