fix: clean up the remote server update reconnect flow

apps/mobile/src/features/review/ReviewSheet.tsx

@@ -771,7 +771,9 @@ export function ReviewSheet(props: ReviewSheetProps) {
                 environment.presentation?.connection ?? {
                   phase: "available",
                   error: null,
+                  reason: null,
                   traceId: null,
+                  retryAt: null,
                 }
               }
               resourceName="review"

apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx

@@ -1209,7 +1209,9 @@ export function ThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps)
               environment.presentation?.connection ?? {
                 phase: "available",
                 error: null,
+                reason: null,
                 traceId: null,
+                retryAt: null,
               }
             }
             resourceName="terminal"

apps/mobile/src/state/workspaceModel.test.ts

@@ -40,7 +40,9 @@ function environment(
     connection: {
       phase,
       error: phase === "error" ? "Connection failed." : null,
+      reason: phase === "error" ? "authentication" : null,
       traceId: phase === "error" ? "trace-1" : null,
+      retryAt: null,
     },
     serverConfig: null,
   };

apps/server/src/cloud/selfUpdate.test.ts

@@ -26,6 +26,30 @@ import * as SelfUpdate from "./selfUpdate.ts";
 
 const NODE_PATH = "/usr/local/bin/node";
 
+/**
+ * The deferred restart runs on a detached fiber, so its rollback performs
+ * real filesystem I/O that advancing the TestClock does not await. Poll on
+ * real time until the expected content lands.
+ */
+const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* (
+  filePath: string,
+  expected: string,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  let last = "";
+  for (let iteration = 0; iteration < 1_000; iteration += 1) {
+    last = yield* fs.readFileString(filePath);
+    if (last === expected) {
+      return last;
+    }
+    // Yielding (rather than sleeping) keeps this on the real event loop:
+    // TestClock would only advance virtual time and never let the
+    // detached fiber's filesystem writes land.
+    yield* Effect.yieldNow;
+  }
+  return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`));
+});
+
 interface RecordedCommand {
   readonly command: string;
   readonly args: ReadonlyArray<string>;
@@ -38,6 +62,8 @@ const makeRecordingRunnerLayer = (
     readonly stdoutFor?:
       | ((command: string, args: ReadonlyArray<string>) => string | undefined)
       | undefined;
+    /** Raises a defect (not a typed failure) to exercise unexpected-error paths. */
+    readonly dieWhen?: ((command: string, args: ReadonlyArray<string>) => boolean) | undefined;
   },
 ) =>
   Layer.succeed(
@@ -46,6 +72,9 @@ const makeRecordingRunnerLayer = (
       run: (input) =>
         Effect.sync(() => {
           commands.push({ command: input.command, args: input.args });
+          if (options?.dieWhen?.(input.command, input.args) === true) {
+            throw new Error(`${input.command} died unexpectedly`);
+          }
           const failed = options?.failWhen?.(input.command, input.args) === true;
           const versionFromPath =
             input.command === NODE_PATH && input.args[1] === "--version"
@@ -257,6 +286,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
     readonly failWhen?: (command: string, args: ReadonlyArray<string>) => boolean;
     readonly stdoutFor?: (command: string, args: ReadonlyArray<string>) => string | undefined;
     readonly failSpawn?: boolean;
+    readonly dieWhen?: (command: string, args: ReadonlyArray<string>) => boolean;
   }) {
     const fs = yield* FileSystem.FileSystem;
     const path = yield* Path.Path;
@@ -329,6 +359,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
           makeRecordingRunnerLayer(commands, {
             failWhen: options?.failWhen,
             stdoutFor: options?.stdoutFor,
+            dieWhen: options?.dieWhen,
           }),
           configLayer,
         ),
@@ -490,7 +521,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
     }).pipe(Effect.provide(TestClock.layer())),
   );
 
-  it.effect("rewrites the systemd unit and restarts the boot service", () =>
+  it.effect("rewrites the systemd unit and defers the boot-service restart", () =>
     Effect.gen(function* () {
       const context = yield* makeContext({ bootService: true });
       const result = yield* context.service.update({ targetVersion: "0.0.29" });
@@ -504,12 +535,15 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
         context.path.join(context.home, ".config", "systemd", "user", "t3code.service"),
       );
       assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`);
+      // The restart is deferred so the RPC acknowledgement flushes before
+      // systemd stops this process.
       assert.deepEqual(
         context.commands.map((entry) => entry.command),
-        ["npm", NODE_PATH, "systemctl", "systemctl"],
+        ["npm", NODE_PATH, "systemctl"],
       );
       assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]);
 
+      yield* TestClock.adjust(Duration.seconds(10));
       assert.deepEqual(context.commands[3], {
         command: "systemctl",
         args: ["--user", "restart", "t3code.service"],
@@ -520,7 +554,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
     }).pipe(Effect.provide(TestClock.layer())),
   );
 
-  it.effect("restores the previous unit and permits a retry when systemd restart fails", () =>
+  it.effect("restores the previous unit and permits a retry when the deferred restart fails", () =>
     Effect.gen(function* () {
       let failRestart = true;
       const context = yield* makeContext({
@@ -542,9 +576,12 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
       );
       const previousUnit = yield* context.fs.readFileString(unitPath);
 
-      const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip);
-      assert.include(first.reason, "Restarting the systemd boot service failed");
-      assert.equal(yield* context.fs.readFileString(unitPath), previousUnit);
+      // The RPC acknowledges before the restart runs, so a rejected
+      // restart can only roll back and log — never fail the request.
+      const first = yield* context.service.update({ targetVersion: "0.0.29" });
+      assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" });
+      yield* TestClock.adjust(Duration.seconds(10));
+      yield* eventuallyFileString(unitPath, previousUnit);
       assert.deepEqual(
         context.commands.slice(-2).map((entry) => entry.args),
         [
@@ -558,6 +595,32 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
     }).pipe(Effect.provide(TestClock.layer())),
   );
 
+  it.effect("permits a retry when the deferred restart dies unexpectedly", () =>
+    Effect.gen(function* () {
+      let dieOnRestart = true;
+      const context = yield* makeContext({
+        bootService: true,
+        dieWhen: (command, args) => {
+          if (command !== "systemctl" || args[1] !== "restart" || !dieOnRestart) {
+            return false;
+          }
+          dieOnRestart = false;
+          return true;
+        },
+      });
+
+      const first = yield* context.service.update({ targetVersion: "0.0.29" });
+      assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" });
+      yield* TestClock.adjust(Duration.seconds(10));
+
+      // A defect bypasses the typed rollback handler, so the in-flight guard
+      // has to be released outside it — otherwise this live process would
+      // reject every further update until restarted by other means.
+      const retry = yield* context.service.update({ targetVersion: "0.0.30" });
+      assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" });
+    }).pipe(Effect.provide(TestClock.layer())),
+  );
+
   it.effect("restores the previous systemd unit when daemon-reload fails", () =>
     Effect.gen(function* () {
       const context = yield* makeContext({

apps/server/src/cloud/selfUpdate.ts

@@ -3,6 +3,7 @@
 // detached fire-and-forget child that outlives this process, while Effect's
 // ChildProcessSpawner ties every child to a scope that kills it.
 import {
+  SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON,
   ServerSelfUpdateError,
   type ServerSelfUpdateCapability,
   type ServerSelfUpdateInput,
@@ -18,6 +19,7 @@ import * as NodeChildProcess from "node:child_process";
 import * as Context from "effect/Context";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
@@ -246,7 +248,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
 
     const alreadyRunning = yield* Ref.getAndSet(inFlight, true);
     if (alreadyRunning) {
-      return yield* failWith("A server update is already in progress.");
+      return yield* failWith(SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON);
     }
 
     return yield* Effect.gen(function* () {
@@ -349,37 +351,61 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
         yield* Effect.logInfo("Server self-update installed; restarting boot service.", {
           targetVersion,
         });
-        // A successful systemd restart stops this process, so the RPC is
-        // interrupted and the reconnecting client observes the new version.
-        // A rejected restart returns while the old process is still alive;
-        // restore the previous unit and report that failure through the RPC.
-        yield* Effect.gen(function* () {
-          const restart = yield* runner
-            .run({
-              command: "systemctl",
-              args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE],
-            })
-            .pipe(
-              Effect.mapError((cause) =>
-                failWith("Could not restart the systemd boot service.", cause),
-              ),
-            );
-          if (restart.code !== 0) {
-            return yield* failWith(
-              `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`,
-            );
-          }
-        }).pipe(
-          Effect.catch((restartError) =>
-            writeUnitAtomically(unitPath, previousUnit).pipe(
-              Effect.andThen(reloadSystemd()),
-              Effect.mapError((rollbackError) =>
-                failWith("Could not restore the previous systemd unit.", {
-                  restartError,
-                  rollbackError,
-                }),
+        // Deferred like the respawn path: a synchronous systemctl restart
+        // stops this process mid-RPC, so every successful update used to
+        // surface to the client as an interrupted call indistinguishable
+        // from a dropped connection. Deferring lets the acknowledgement
+        // flush first. The trade: a rejected restart can no longer be
+        // reported through this RPC — it rolls back, logs, and clears the
+        // in-flight guard so a retry works.
+        yield* scheduleRestart(
+          Effect.gen(function* () {
+            const restart = yield* runner
+              .run({
+                command: "systemctl",
+                args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE],
+              })
+              .pipe(
+                Effect.mapError((cause) =>
+                  failWith("Could not restart the systemd boot service.", cause),
+                ),
+              );
+            if (restart.code !== 0) {
+              return yield* failWith(
+                `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`,
+              );
+            }
+          }).pipe(
+            Effect.catch((restartError) =>
+              Effect.logError(
+                "Server self-update restart was rejected; restoring the previous systemd unit.",
+              ).pipe(
+                Effect.annotateLogs({ targetVersion, error: restartError.reason }),
+                Effect.andThen(
+                  writeUnitAtomically(unitPath, previousUnit).pipe(Effect.andThen(reloadSystemd())),
+                ),
+                Effect.catch((rollbackError) =>
+                  Effect.logError(
+                    "Server self-update could not restore the previous systemd unit after a rejected restart.",
+                  ).pipe(Effect.annotateLogs({ targetVersion, error: String(rollbackError) })),
+                ),
+                // The process survived a rejected restart, so it must stay
+                // updatable — leaving the guard set would reject every retry
+                // until the server was restarted by other means.
+                Effect.ensuring(Ref.set(inFlight, false)),
               ),
-              Effect.andThen(Effect.fail(restartError)),
+            ),
+            // The typed catch above cannot see defects or interrupts, which
+            // would otherwise strand the guard on a live process. A success
+            // deliberately leaves it set: systemd is about to stop this
+            // process and a second update would race the handoff.
+            Effect.onExit((exit) =>
+              Exit.isSuccess(exit)
+                ? Effect.void
+                : Effect.logError("Server self-update restart fiber ended unexpectedly.").pipe(
+                    Effect.annotateLogs({ targetVersion, exit: String(exit) }),
+                    Effect.ensuring(Ref.set(inFlight, false)),
+                  ),
             ),
           ),
         );