fix(daemon): repair virtual plist paths, sweep orphans, launchd-exclusive start

Root cause of 'daemon starts then exits' on compiled-binary installs:
the launchd plist captured process.argv[1] from inside the compiled
Bun binary — the virtual /$bunfs/root/hack path, which passes
pathExists inside that process but is unexecutable by launchd (exit
status 78, flapping forever). Reproduced and fixed live on this
machine. The bin-path resolver now rejects virtual paths and prefers
stable locations; daemon start self-repairs invalid plists; daemons
that outlived their pid file (invisible to status/clear/start) are
detected and swept; an incompatible running daemon is replaced instead
of reported as success; and when launchd manages the service, start
goes through launchd exclusively — never a bare spawn beside it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: roodboi

src/commands/daemon.ts

@@ -14,10 +14,16 @@ import {
   installLaunchdService,
   kickstartLaunchdService,
   type LaunchdServiceStatus,
+  repairLaunchdProgramIfInvalid,
   uninstallLaunchdService,
 } from "../daemon/launchd.ts";
 import { type DaemonPaths, resolveDaemonPaths } from "../daemon/paths.ts";
-import { removeFileIfExists, waitForProcessExit } from "../daemon/process.ts";
+import {
+  findOrphanDaemonProcesses,
+  removeFileIfExists,
+  terminateOrphanDaemonProcesses,
+  waitForProcessExit,
+} from "../daemon/process.ts";
 import { runDaemon } from "../daemon/server.ts";
 import {
   buildDaemonRepairMessage,
@@ -220,11 +226,38 @@ async function handleDaemonStart({
   const paths = resolveDaemonPaths({});
   const status = await readDaemonStatus({ paths });
 
-  if (status.running) {
-    logger.success({
-      message: `hackd already running (pid ${status.pid ?? "unknown"})`,
+  if (status.running && status.pid !== null) {
+    const api = await checkDaemonApi({
+      socketExists: status.socketExists,
+      paths,
     });
-    return 0;
+    if (api.reachable && !api.compatible) {
+      // An incompatible daemon (usually a pre-upgrade binary) must be
+      // replaced, not reported as success — leaving it running is how
+      // machines end up with doctor/status contradictions.
+      logger.warn({
+        message: `Replacing incompatible hackd (pid ${status.pid})`,
+      });
+      await stopDaemonProcess({ pid: status.pid, paths });
+    } else {
+      logger.success({
+        message: `hackd already running (pid ${status.pid})`,
+      });
+      return 0;
+    }
+  }
+
+  // Daemons that outlived their pid file (e.g. a launchd-managed instance
+  // surviving a manual clear/restart) hold the API socket invisibly and
+  // make every freshly spawned daemon exit cleanly. Sweep them first.
+  const orphans = await findOrphanDaemonProcesses({
+    trackedPid: status.pid,
+  });
+  if (orphans.length > 0) {
+    logger.warn({
+      message: `Stopping orphaned hackd process(es) not tracked by the pid file: ${orphans.join(", ")}`,
+    });
+    await terminateOrphanDaemonProcesses({ pids: orphans });
   }
 
   await removeFileIfExists({ path: paths.socketPath });
@@ -235,6 +268,43 @@ async function handleDaemonStart({
     return 0;
   }
 
+  // When launchd manages the daemon, start through launchd — spawning a
+  // bare process next to a loaded agent means two process managers fight
+  // over one socket and pid file.
+  const launchdStatus = await resolveLaunchdStatus({ paths });
+  if (launchdStatus?.loaded) {
+    const repair = await repairLaunchdProgramIfInvalid({ paths });
+    if (repair === "repaired") {
+      logger.warn({
+        message:
+          "Repaired launchd service: its program path was invalid (stale or compiled-binary virtual path)",
+      });
+    }
+    // launchd owns the daemon: never spawn a bare process next to it —
+    // two process managers for one socket is how machines end up with
+    // start-then-exit flapping and stale-pid contradictions.
+    const kick = await kickstartLaunchdService();
+    if (!kick.ok) {
+      logger.error({
+        message: `launchd kickstart failed: ${kick.error ?? "unknown error"} | Check: hack daemon logs --tail 200`,
+      });
+      return 1;
+    }
+    const startedViaLaunchd = await waitForDaemonStart({
+      paths,
+      timeoutMs: 8000,
+    });
+    if (startedViaLaunchd) {
+      logger.success({ message: "hackd started (launchd)" });
+      return 0;
+    }
+    logger.warn({
+      message:
+        "launchd kickstarted hackd but it did not report ready yet | Check: hack daemon logs --tail 200",
+    });
+    return 1;
+  }
+
   const invocation = await resolveHackInvocation();
   const cmd = [...invocation.args, "daemon", "start", "--foreground"];
   const proc = Bun.spawn([invocation.bin, ...cmd], {
@@ -256,6 +326,31 @@ async function handleDaemonStart({
   return 0;
 }
 
+async function stopDaemonProcess(opts: {
+  readonly pid: number;
+  readonly paths: DaemonPaths;
+}): Promise<void> {
+  try {
+    process.kill(opts.pid, "SIGTERM");
+  } catch {
+    // Already gone.
+  }
+  const exited = await waitForProcessExit({
+    pid: opts.pid,
+    timeoutMs: 2000,
+    pollMs: 200,
+  });
+  if (!exited) {
+    try {
+      process.kill(opts.pid, "SIGKILL");
+    } catch {
+      // Already gone.
+    }
+  }
+  await removeFileIfExists({ path: opts.paths.pidPath });
+  await removeFileIfExists({ path: opts.paths.socketPath });
+}
+
 async function handleDaemonStop({
   args: _args,
 }: {
@@ -506,11 +601,26 @@ async function handleDaemonClear({
     return 1;
   }
 
+  const orphans = await findOrphanDaemonProcesses({
+    trackedPid: status.pid,
+  });
+  if (orphans.length > 0) {
+    logger.warn({
+      message: `Stopping orphaned hackd process(es): ${orphans.join(", ")}`,
+    });
+    await terminateOrphanDaemonProcesses({ pids: orphans });
+  }
+
   const pidExists = await pathExists(paths.pidPath);
   const socketExists = await pathExists(paths.socketPath);
 
   if (!(pidExists || socketExists)) {
-    logger.info({ message: "No stale hackd state found" });
+    logger.info({
+      message:
+        orphans.length > 0
+          ? "Cleared orphaned hackd process(es); no stale files found"
+          : "No stale hackd state found",
+    });
     return 0;
   }
 
@@ -549,10 +659,12 @@ async function handleDaemonRestart({
 
 async function waitForDaemonStart({
   paths,
+  timeoutMs,
 }: {
   readonly paths: ReturnType<typeof resolveDaemonPaths>;
+  readonly timeoutMs?: number;
 }): Promise<boolean> {
-  const deadline = Date.now() + 2000;
+  const deadline = Date.now() + (timeoutMs ?? 2000);
   while (Date.now() < deadline) {
     const status = await readDaemonStatus({ paths });
     if (status.running && status.socketExists) {

src/daemon/launchd.ts

@@ -167,12 +167,16 @@ async function resolveLaunchdHackBinPath(opts: {
     readonly args: readonly string[];
   };
 }): Promise<string | null> {
+  // Stable locations first: versioned paths (homebrew Cellar) rot on
+  // upgrade, and argv paths inside a compiled Bun binary report the
+  // virtual /$bunfs mount, which only exists inside that process —
+  // launchd exec'ing it fails forever with status 78.
   const candidates = [
-    process.argv[1] ?? null,
+    resolve(resolveGlobalHackDir(), "bin", "hack"),
+    await findExecutableInPath("hack"),
     opts.invocation.args[0] ?? null,
     opts.invocation.bin,
-    await findExecutableInPath("hack"),
-    resolve(resolveGlobalHackDir(), "bin", "hack"),
+    process.argv[1] ?? null,
   ];
 
   for (const raw of candidates) {
@@ -192,13 +196,78 @@ function normalizeHackExecutablePath(raw: string | null): string | null {
   if (!trimmed) {
     return null;
   }
+  if (isVirtualExecutablePath(trimmed)) {
+    return null;
+  }
   const base = basename(trimmed).toLowerCase();
   if (!(base === "hack" || base.startsWith("hack-"))) {
     return null;
   }
   return trimmed;
 }
 
+/**
+ * Paths inside a compiled Bun binary's virtual filesystem. They pass
+ * pathExists from INSIDE the same process but no other process (launchd
+ * included) can exec them.
+ */
+export function isVirtualExecutablePath(path: string): boolean {
+  return path.startsWith("/$bunfs/") || path.includes("~BUN");
+}
+
+/**
+ * Extracts ProgramArguments[0] from rendered launchd plist text.
+ */
+export function extractLaunchdProgramPath(opts: {
+  readonly plistText: string;
+}): string | null {
+  const anchor = opts.plistText.indexOf("<key>ProgramArguments</key>");
+  if (anchor < 0) {
+    return null;
+  }
+  const match = opts.plistText.slice(anchor).match(/<string>([^<]+)<\/string>/);
+  return match?.[1] ?? null;
+}
+
+export type LaunchdProgramRepairResult =
+  | "ok"
+  | "repaired"
+  | "not-installed"
+  | "failed";
+
+/**
+ * Repairs an installed launchd plist whose program path is invalid — a
+ * compiled-binary virtual path (/$bunfs) or a binary that no longer exists
+ * (e.g. an upgraded homebrew Cellar path). Broken plists leave launchd
+ * flapping with exit status 78 and the daemon unable to stay up.
+ */
+export async function repairLaunchdProgramIfInvalid(opts: {
+  readonly paths: DaemonPaths;
+}): Promise<LaunchdProgramRepairResult> {
+  const plistText = await readTextFile(opts.paths.launchdPlistPath);
+  if (plistText === null) {
+    return "not-installed";
+  }
+  const program = extractLaunchdProgramPath({ plistText });
+  const valid =
+    program !== null &&
+    !isVirtualExecutablePath(program) &&
+    (await pathExists(program));
+  if (valid) {
+    return "ok";
+  }
+
+  const result = await installLaunchdService({
+    paths: opts.paths,
+    config: {
+      installed: true,
+      runAtLoad: plistText.includes("<key>RunAtLoad</key>\n  <true/>"),
+      guiSessionOnly: plistText.includes("LimitLoadToSessionType"),
+    },
+  });
+  return result.ok ? "repaired" : "failed";
+}
+
 export interface LaunchdUninstallResult {
   readonly ok: boolean;
   readonly error?: string;

src/daemon/process.ts

@@ -66,3 +66,83 @@ export async function waitForProcessExit({
 function sleep({ ms }: { readonly ms: number }): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+const DAEMON_COMMAND_MARKER = "daemon start --foreground";
+
+/**
+ * Finds hackd processes that the pid file does not track ("orphans").
+ *
+ * Orphans appear when a daemon keeps running after its pid/socket files were
+ * replaced or cleared (e.g. a launchd-managed instance surviving a manual
+ * restart). They are invisible to status/start/clear, which only consult the
+ * pid file — the root cause of "daemon starts then exits" contradictions:
+ * the orphan holds the API while every newly spawned daemon exits.
+ *
+ * @param opts.trackedPid - pid currently recorded in the pid file, if any.
+ * @param opts.psLines - injectable `ps -axo pid=,command=` lines for tests.
+ */
+export async function findOrphanDaemonProcesses(opts: {
+  readonly trackedPid: number | null;
+  readonly psLines?: readonly string[];
+}): Promise<readonly number[]> {
+  const lines = opts.psLines ?? (await listProcessTable());
+  const orphans: number[] = [];
+  for (const line of lines) {
+    const trimmed = line.trim();
+    const spaceIdx = trimmed.indexOf(" ");
+    if (spaceIdx <= 0) {
+      continue;
+    }
+    const pid = Number.parseInt(trimmed.slice(0, spaceIdx), 10);
+    const command = trimmed.slice(spaceIdx + 1);
+    if (!Number.isFinite(pid) || pid <= 0) {
+      continue;
+    }
+    if (!command.includes(DAEMON_COMMAND_MARKER)) {
+      continue;
+    }
+    if (pid === opts.trackedPid || pid === process.pid) {
+      continue;
+    }
+    orphans.push(pid);
+  }
+  return orphans;
+}
+
+async function listProcessTable(): Promise<readonly string[]> {
+  const proc = Bun.spawn(["ps", "-axo", "pid=,command="], {
+    stdout: "pipe",
+    stderr: "ignore",
+    stdin: "ignore",
+  });
+  const text = await new Response(proc.stdout).text();
+  await proc.exited;
+  return text.split("\n");
+}
+
+/**
+ * Terminates orphan daemon processes (SIGTERM, bounded wait). Returns the
+ * pids that were actually terminated.
+ */
+export async function terminateOrphanDaemonProcesses(opts: {
+  readonly pids: readonly number[];
+  readonly timeoutMs?: number;
+}): Promise<readonly number[]> {
+  const terminated: number[] = [];
+  for (const pid of opts.pids) {
+    try {
+      process.kill(pid, "SIGTERM");
+    } catch {
+      continue;
+    }
+    const exited = await waitForProcessExit({
+      pid,
+      timeoutMs: opts.timeoutMs ?? 3000,
+      pollMs: 100,
+    });
+    if (exited) {
+      terminated.push(pid);
+    }
+  }
+  return terminated;
+}