diff --git a/apps/mobile/src/features/review/ReviewSheet.tsx b/apps/mobile/src/features/review/ReviewSheet.tsx index 1ebb3aaf7b1..82269bade54 100644 --- a/apps/mobile/src/features/review/ReviewSheet.tsx +++ b/apps/mobile/src/features/review/ReviewSheet.tsx @@ -771,7 +771,9 @@ export function ReviewSheet(props: ReviewSheetProps) { environment.presentation?.connection ?? { phase: "available", error: null, + reason: null, traceId: null, + retryAt: null, } } resourceName="review" diff --git a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx index cb281bf4aed..5192599915b 100644 --- a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx +++ b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx @@ -1209,7 +1209,9 @@ export function ThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps) environment.presentation?.connection ?? { phase: "available", error: null, + reason: null, traceId: null, + retryAt: null, } } resourceName="terminal" diff --git a/apps/mobile/src/state/workspaceModel.test.ts b/apps/mobile/src/state/workspaceModel.test.ts index e51273d57de..0fe2c2c4b7b 100644 --- a/apps/mobile/src/state/workspaceModel.test.ts +++ b/apps/mobile/src/state/workspaceModel.test.ts @@ -40,7 +40,9 @@ function environment( connection: { phase, error: phase === "error" ? "Connection failed." : null, + reason: phase === "error" ? "authentication" : null, traceId: phase === "error" ? "trace-1" : null, + retryAt: null, }, serverConfig: null, }; diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 9d6e3801704..a81649e07f7 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -26,6 +26,30 @@ import * as SelfUpdate from "./selfUpdate.ts"; const NODE_PATH = "/usr/local/bin/node"; +/** + * The deferred restart runs on a detached fiber, so its rollback performs + * real filesystem I/O that advancing the TestClock does not await. Poll on + * real time until the expected content lands. + */ +const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* ( + filePath: string, + expected: string, +) { + const fs = yield* FileSystem.FileSystem; + let last = ""; + for (let iteration = 0; iteration < 1_000; iteration += 1) { + last = yield* fs.readFileString(filePath); + if (last === expected) { + return last; + } + // Yielding (rather than sleeping) keeps this on the real event loop: + // TestClock would only advance virtual time and never let the + // detached fiber's filesystem writes land. + yield* Effect.yieldNow; + } + return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`)); +}); + interface RecordedCommand { readonly command: string; readonly args: ReadonlyArray; @@ -38,6 +62,8 @@ const makeRecordingRunnerLayer = ( readonly stdoutFor?: | ((command: string, args: ReadonlyArray) => string | undefined) | undefined; + /** Raises a defect (not a typed failure) to exercise unexpected-error paths. */ + readonly dieWhen?: ((command: string, args: ReadonlyArray) => boolean) | undefined; }, ) => Layer.succeed( @@ -46,6 +72,9 @@ const makeRecordingRunnerLayer = ( run: (input) => Effect.sync(() => { commands.push({ command: input.command, args: input.args }); + if (options?.dieWhen?.(input.command, input.args) === true) { + throw new Error(`${input.command} died unexpectedly`); + } const failed = options?.failWhen?.(input.command, input.args) === true; const versionFromPath = input.command === NODE_PATH && input.args[1] === "--version" @@ -257,6 +286,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { readonly failWhen?: (command: string, args: ReadonlyArray) => boolean; readonly stdoutFor?: (command: string, args: ReadonlyArray) => string | undefined; readonly failSpawn?: boolean; + readonly dieWhen?: (command: string, args: ReadonlyArray) => boolean; }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -329,6 +359,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { makeRecordingRunnerLayer(commands, { failWhen: options?.failWhen, stdoutFor: options?.stdoutFor, + dieWhen: options?.dieWhen, }), configLayer, ), @@ -490,7 +521,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("rewrites the systemd unit and restarts the boot service", () => + it.effect("rewrites the systemd unit and defers the boot-service restart", () => Effect.gen(function* () { const context = yield* makeContext({ bootService: true }); const result = yield* context.service.update({ targetVersion: "0.0.29" }); @@ -504,12 +535,15 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { context.path.join(context.home, ".config", "systemd", "user", "t3code.service"), ); assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); + // The restart is deferred so the RPC acknowledgement flushes before + // systemd stops this process. assert.deepEqual( context.commands.map((entry) => entry.command), - ["npm", NODE_PATH, "systemctl", "systemctl"], + ["npm", NODE_PATH, "systemctl"], ); assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]); + yield* TestClock.adjust(Duration.seconds(10)); assert.deepEqual(context.commands[3], { command: "systemctl", args: ["--user", "restart", "t3code.service"], @@ -520,7 +554,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { }).pipe(Effect.provide(TestClock.layer())), ); - it.effect("restores the previous unit and permits a retry when systemd restart fails", () => + it.effect("restores the previous unit and permits a retry when the deferred restart fails", () => Effect.gen(function* () { let failRestart = true; const context = yield* makeContext({ @@ -542,9 +576,12 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { ); const previousUnit = yield* context.fs.readFileString(unitPath); - const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(first.reason, "Restarting the systemd boot service failed"); - assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); + // The RPC acknowledges before the restart runs, so a rejected + // restart can only roll back and log — never fail the request. + const first = yield* context.service.update({ targetVersion: "0.0.29" }); + assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); + yield* TestClock.adjust(Duration.seconds(10)); + yield* eventuallyFileString(unitPath, previousUnit); assert.deepEqual( context.commands.slice(-2).map((entry) => entry.args), [ @@ -558,6 +595,32 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("permits a retry when the deferred restart dies unexpectedly", () => + Effect.gen(function* () { + let dieOnRestart = true; + const context = yield* makeContext({ + bootService: true, + dieWhen: (command, args) => { + if (command !== "systemctl" || args[1] !== "restart" || !dieOnRestart) { + return false; + } + dieOnRestart = false; + return true; + }, + }); + + const first = yield* context.service.update({ targetVersion: "0.0.29" }); + assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); + yield* TestClock.adjust(Duration.seconds(10)); + + // A defect bypasses the typed rollback handler, so the in-flight guard + // has to be released outside it — otherwise this live process would + // reject every further update until restarted by other means. + const retry = yield* context.service.update({ targetVersion: "0.0.30" }); + assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" }); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("restores the previous systemd unit when daemon-reload fails", () => Effect.gen(function* () { const context = yield* makeContext({ diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 62bd07fbbc8..f8d0b532b2b 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -3,6 +3,7 @@ // detached fire-and-forget child that outlives this process, while Effect's // ChildProcessSpawner ties every child to a scope that kills it. import { + SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON, ServerSelfUpdateError, type ServerSelfUpdateCapability, type ServerSelfUpdateInput, @@ -18,6 +19,7 @@ import * as NodeChildProcess from "node:child_process"; import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; @@ -246,7 +248,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option const alreadyRunning = yield* Ref.getAndSet(inFlight, true); if (alreadyRunning) { - return yield* failWith("A server update is already in progress."); + return yield* failWith(SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON); } return yield* Effect.gen(function* () { @@ -349,37 +351,61 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option yield* Effect.logInfo("Server self-update installed; restarting boot service.", { targetVersion, }); - // A successful systemd restart stops this process, so the RPC is - // interrupted and the reconnecting client observes the new version. - // A rejected restart returns while the old process is still alive; - // restore the previous unit and report that failure through the RPC. - yield* Effect.gen(function* () { - const restart = yield* runner - .run({ - command: "systemctl", - args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], - }) - .pipe( - Effect.mapError((cause) => - failWith("Could not restart the systemd boot service.", cause), - ), - ); - if (restart.code !== 0) { - return yield* failWith( - `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`, - ); - } - }).pipe( - Effect.catch((restartError) => - writeUnitAtomically(unitPath, previousUnit).pipe( - Effect.andThen(reloadSystemd()), - Effect.mapError((rollbackError) => - failWith("Could not restore the previous systemd unit.", { - restartError, - rollbackError, - }), + // Deferred like the respawn path: a synchronous systemctl restart + // stops this process mid-RPC, so every successful update used to + // surface to the client as an interrupted call indistinguishable + // from a dropped connection. Deferring lets the acknowledgement + // flush first. The trade: a rejected restart can no longer be + // reported through this RPC — it rolls back, logs, and clears the + // in-flight guard so a retry works. + yield* scheduleRestart( + Effect.gen(function* () { + const restart = yield* runner + .run({ + command: "systemctl", + args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + }) + .pipe( + Effect.mapError((cause) => + failWith("Could not restart the systemd boot service.", cause), + ), + ); + if (restart.code !== 0) { + return yield* failWith( + `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`, + ); + } + }).pipe( + Effect.catch((restartError) => + Effect.logError( + "Server self-update restart was rejected; restoring the previous systemd unit.", + ).pipe( + Effect.annotateLogs({ targetVersion, error: restartError.reason }), + Effect.andThen( + writeUnitAtomically(unitPath, previousUnit).pipe(Effect.andThen(reloadSystemd())), + ), + Effect.catch((rollbackError) => + Effect.logError( + "Server self-update could not restore the previous systemd unit after a rejected restart.", + ).pipe(Effect.annotateLogs({ targetVersion, error: String(rollbackError) })), + ), + // The process survived a rejected restart, so it must stay + // updatable — leaving the guard set would reject every retry + // until the server was restarted by other means. + Effect.ensuring(Ref.set(inFlight, false)), ), - Effect.andThen(Effect.fail(restartError)), + ), + // The typed catch above cannot see defects or interrupts, which + // would otherwise strand the guard on a live process. A success + // deliberately leaves it set: systemd is about to stop this + // process and a second update would race the handoff. + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? Effect.void + : Effect.logError("Server self-update restart fiber ended unexpectedly.").pipe( + Effect.annotateLogs({ targetVersion, exit: String(exit) }), + Effect.ensuring(Ref.set(inFlight, false)), + ), ), ), ); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index ab1256cddb3..c75a35dbcc1 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -25,6 +25,11 @@ import { connectionStatusTitle, type EnvironmentConnectionPresentation, } from "@t3tools/client-runtime/connection"; +import { + clearExpectedServerRestart, + isServerRestarting, + useServerRestartExpected, +} from "~/serverRestartStore"; import { effectiveSettled, effectiveSnoozed } from "@t3tools/client-runtime/state/thread-settled"; import { parseScopedThreadKey, @@ -275,6 +280,7 @@ import { RightPanelSheet } from "./RightPanelSheet"; import { previewEnvironment } from "../state/preview"; import { useAtomCommand } from "../state/use-atom-command"; import { Button } from "./ui/button"; +import { Spinner } from "./ui/spinner"; import { AlertDialog, AlertDialogClose, @@ -1619,6 +1625,54 @@ function ChatViewContent(props: ChatViewProps) { connection: activeEnvironment.connection, }; }, [activeEnvironment, activeEnvironmentUnavailable, activeEnvironmentUnavailableLabel]); + // While a server self-update restart is expected, disconnects are routine: + // present them calmly and retry fast instead of letting exponential + // backoff strand the client asleep when the server comes back. + const activeEnvironmentId = activeEnvironment?.environmentId ?? null; + const activeEnvironmentRestartExpected = useServerRestartExpected(activeEnvironmentId); + const activeEnvironmentRestarting = isServerRestarting({ + restartExpected: activeEnvironmentRestartExpected, + environmentUnavailable: activeEnvironmentUnavailable, + }); + // While the restart is expected, poke the supervisor on a flat cadence so a + // 16-second backoff sleep cannot outlast the server's return. + // + // The window also has to end at the right moment. Ending it on any healthy + // connection is wrong — it is armed while the old server is still connected, + // so that clears it before the restart it exists to cover. Never ending it + // is also wrong — an unrelated outage later in the window would be dressed + // up as an intentional restart. So it ends once this environment has gone + // away and come back: the restart it was armed for has been seen through. + const restartSawDisconnectRef = useRef(false); + useEffect(() => { + if (activeEnvironmentId === null || !activeEnvironmentRestartExpected) { + restartSawDisconnectRef.current = false; + return; + } + if (activeEnvironmentConnectionPhase === "connected") { + if (restartSawDisconnectRef.current) { + restartSawDisconnectRef.current = false; + clearExpectedServerRestart(activeEnvironmentId); + } + return; + } + restartSawDisconnectRef.current = true; + // Only wake the supervisor out of a backoff sleep; while an attempt is + // live, retryNow would interrupt and restart it. + if (activeEnvironment?.connection.retryAt == null) { + return; + } + const timer = setTimeout(() => { + void retryEnvironment(activeEnvironmentId); + }, 2_000); + return () => clearTimeout(timer); + }, [ + activeEnvironment?.connection.retryAt, + activeEnvironmentConnectionPhase, + activeEnvironmentId, + activeEnvironmentRestartExpected, + retryEnvironment, + ]); const handleReconnectActiveEnvironment = useCallback( async (environmentId: EnvironmentId) => { const result = await retryEnvironment(environmentId); @@ -1832,7 +1886,19 @@ function ChatViewContent(props: ChatViewProps) { const versionMismatchSelfUpdate = resolveServerSelfUpdateCapability(serverConfig); const systemComposerBannerItems = useMemo(() => { const items: ComposerBannerStackItem[] = []; - if (activeEnvironmentUnavailableState) { + if (activeEnvironmentUnavailableState && activeEnvironmentRestarting) { + // An expected self-update restart: the disconnect and the transient + // connection errors behind it are routine, so keep the tone neutral + // and let the fast auto-retry loop do the work instead of asking the + // user to reconnect by hand. + items.push({ + id: `environment-restarting:${activeEnvironmentUnavailableState.environmentId}`, + variant: "info", + icon: , + title: `${activeEnvironmentUnavailableState.label} is restarting`, + description: "Finishing the server update — reconnecting automatically.", + }); + } else if (activeEnvironmentUnavailableState) { const connection = activeEnvironmentUnavailableState.connection; const isReconnecting = connection.phase === "connecting" || connection.phase === "reconnecting"; @@ -1872,7 +1938,10 @@ function ChatViewContent(props: ChatViewProps) { showVersionMismatchBanner && versionMismatch && versionMismatchDismissKey && - versionMismatchEnvironmentId + versionMismatchEnvironmentId && + // The mismatch data predates the disconnect; during an expected + // restart it is stale and its update action would double-trigger. + !activeEnvironmentRestarting ) { items.push({ id: `version-mismatch:${versionMismatchDismissKey}`, @@ -1906,6 +1975,7 @@ function ChatViewContent(props: ChatViewProps) { } return items; }, [ + activeEnvironmentRestarting, activeEnvironmentUnavailableState, handleReconnectActiveEnvironment, navigate, diff --git a/apps/web/src/components/ServerUpdateAction.test.tsx b/apps/web/src/components/ServerUpdateAction.test.tsx index c0ba1c693d4..093b22a5136 100644 --- a/apps/web/src/components/ServerUpdateAction.test.tsx +++ b/apps/web/src/components/ServerUpdateAction.test.tsx @@ -2,13 +2,27 @@ import type { Dispatch, ReactElement, SetStateAction } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; import * as Cause from "effect/Cause"; import { AsyncResult } from "effect/unstable/reactivity"; -import type { EnvironmentId } from "@t3tools/contracts"; +import { + SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON, + ServerSelfUpdateError, + type EnvironmentId, +} from "@t3tools/contracts"; const testState = vi.hoisted(() => ({ updateServer: vi.fn(), toast: vi.fn(), })); +const restartStore = vi.hoisted(() => ({ + expect: vi.fn(), + clear: vi.fn(), + expected: vi.fn(() => false), +})); + +const environmentState = vi.hoisted(() => ({ + useEnvironment: vi.fn(() => ({ connection: { phase: "connected" } })), +})); + const hooks = vi.hoisted(() => { let cursor = 0; let slots: unknown[] = []; @@ -78,6 +92,18 @@ vi.mock("~/state/use-atom-command", () => ({ vi.mock("./ui/toast", () => ({ toastManager: { add: testState.toast }, })); +vi.mock("~/serverRestartStore", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + expectServerRestart: restartStore.expect, + clearExpectedServerRestart: restartStore.clear, + useServerRestartExpected: restartStore.expected, + }; +}); +vi.mock("~/state/environments", () => ({ + useEnvironment: environmentState.useEnvironment, +})); import { ServerUpdateAction } from "./ServerUpdateAction"; @@ -118,6 +144,12 @@ describe("ServerUpdateAction", () => { hooks.reset(); testState.updateServer.mockReset(); testState.toast.mockReset(); + restartStore.expect.mockReset(); + restartStore.clear.mockReset(); + restartStore.expected.mockReset(); + restartStore.expected.mockReturnValue(false); + environmentState.useEnvironment.mockReset(); + environmentState.useEnvironment.mockReturnValue({ connection: { phase: "connected" } }); }); afterEach(() => { @@ -195,4 +227,71 @@ describe("ServerUpdateAction", () => { expect.objectContaining({ title: "Server update failed" }), ); }); + + it("re-arms the restart window when a slow install finally succeeds", async () => { + const update = + deferred< + ReturnType> + >(); + testState.updateServer.mockReturnValue(update.promise); + + renderAction().props.onClick?.(); + expect(restartStore.expect).toHaveBeenCalledTimes(1); + + // An install can outlast the 90s restart window armed at click time, so + // the acknowledgement must restart the clock or the ensuing disconnect + // would be presented as an outage. + await vi.advanceTimersByTimeAsync(5 * 60_000); + update.resolve(AsyncResult.success({ targetVersion: "0.0.29", method: "boot-service" })); + await flushPromises(); + + expect(restartStore.expect).toHaveBeenCalledTimes(2); + expect(restartStore.clear).not.toHaveBeenCalled(); + }); + + it("re-arms the restart window on an interrupted restart RPC", async () => { + testState.updateServer.mockResolvedValue(AsyncResult.failure(Cause.interrupt())); + + renderAction().props.onClick?.(); + await flushPromises(); + + expect(restartStore.expect).toHaveBeenCalledTimes(2); + expect(restartStore.clear).not.toHaveBeenCalled(); + }); + + it("withdraws the restart expectation when the server reports a real failure", async () => { + testState.updateServer.mockResolvedValue( + AsyncResult.failure( + Cause.fail( + new ServerSelfUpdateError({ reason: "Could not install the requested t3 version." }), + ), + ), + ); + + renderAction().props.onClick?.(); + await flushPromises(); + + // The server answered, so it is still alive and no restart is coming. + expect(restartStore.clear).toHaveBeenCalledTimes(1); + expect(testState.toast).toHaveBeenCalledWith( + expect.objectContaining({ title: "Server update failed" }), + ); + }); + + it("keeps the restart expectation when an update is already in progress", async () => { + testState.updateServer.mockResolvedValue( + AsyncResult.failure( + Cause.fail( + new ServerSelfUpdateError({ reason: SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON }), + ), + ), + ); + + renderAction().props.onClick?.(); + await flushPromises(); + + // An earlier request is still installing, so a restart really is pending + // — clearing here would restore the harsh reconnect UI mid-restart. + expect(restartStore.clear).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/src/components/ServerUpdateAction.tsx b/apps/web/src/components/ServerUpdateAction.tsx index 5e82c4e4d93..e10bfaab648 100644 --- a/apps/web/src/components/ServerUpdateAction.tsx +++ b/apps/web/src/components/ServerUpdateAction.tsx @@ -1,11 +1,23 @@ import { useEffect, useRef, useState } from "react"; -import type { EnvironmentId, ServerSelfUpdateCapability } from "@t3tools/contracts"; +import { + ServerSelfUpdateError, + type EnvironmentId, + type ServerSelfUpdateCapability, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; import { isAtomCommandInterrupted, squashAtomCommandFailure, } from "@t3tools/client-runtime/state/runtime"; import { useCopyToClipboard } from "~/hooks/useCopyToClipboard"; +import { + clearExpectedServerRestart, + expectServerRestart, + isExpectedRestartResolved, + useServerRestartExpected, +} from "~/serverRestartStore"; +import { useEnvironment } from "~/state/environments"; import { serverEnvironment } from "~/state/server"; import { useAtomCommand } from "~/state/use-atom-command"; import { manualServerUpdateCommand } from "~/versionSkew"; @@ -24,6 +36,18 @@ function updateFailureMessage(error: unknown): string { return error instanceof Error ? error.message : "Server update failed."; } +/** + * The server rejects a second update while one is already installing. That + * failure means a restart is still pending from the earlier request, so the + * restart expectation must survive it — unlike every other failure, which + * proves nothing is coming. + */ +const isServerSelfUpdateError = Schema.is(ServerSelfUpdateError); + +function isUpdateAlreadyRunning(error: unknown): boolean { + return isServerSelfUpdateError(error) && error.isAlreadyRunning; +} + /** * The call-to-action for a version-skewed server, matched to the update path * it advertises: a one-click install-and-restart for servers that can update @@ -80,6 +104,56 @@ export function ServerUpdateAction({ [], ); + // The restart is scheduled after the RPC is acknowledged, so a rejected + // restart — or a replacement that comes back on the old version — can no + // longer be reported through the call. Without a release path the mismatch + // banner would keep its only action disabled for the full safety window. + // + // The signal is the environment going away and then answering again while + // this component is still mounted: still mounted means the skew is + // unresolved, so whatever restarted did not deliver the new version. + // Deliberately independent of the restart window, which is UI presentation + // state and can lapse or be re-armed for unrelated reasons. + // Two ways that resolves, because a rejected restart may or may not have + // taken the connection down with it: + // - the environment went away and came back, so something restarted but + // did not deliver the new version; or + // - the restart window lapsed with the connection never dropping, so no + // restart ever happened (systemd refused it and rolled back). + const connectionPhase = useEnvironment(environmentId)?.connection.phase ?? null; + const connectionUnavailable = connectionPhase !== null && connectionPhase !== "connected"; + const restartExpected = useServerRestartExpected(environmentId); + const wentAwayRef = useRef(false); + const sawWindowRef = useRef(false); + if (connectionUnavailable) { + wentAwayRef.current = true; + } + if (restartExpected) { + // Guards against reading a not-yet-rendered window as "already lapsed" + // in the render between the click and the store update landing. + sawWindowRef.current = true; + } + const restartResolved = isExpectedRestartResolved({ + sawRestartWindow: sawWindowRef.current, + restartExpected, + wentAway: wentAwayRef.current, + environmentUnavailable: connectionUnavailable, + }); + useEffect(() => { + if (!restartResolved || !inFlightRef.current) { + return; + } + wentAwayRef.current = false; + sawWindowRef.current = false; + if (expiryRef.current !== null) { + clearTimeout(expiryRef.current); + expiryRef.current = null; + } + attemptRef.current += 1; + inFlightRef.current = false; + setPending(false); + }, [restartResolved]); + const handleUpdate = () => { // Synchronous re-entry guard: setPending is async, so a rapid // double-click would otherwise dispatch two updates. @@ -87,6 +161,11 @@ export function ServerUpdateAction({ return; } inFlightRef.current = true; + // Only outages observed during this attempt may resolve it. An earlier + // unrelated blip would otherwise satisfy the resolve predicate on the + // very next render and re-enable the button mid-update. + wentAwayRef.current = false; + sawWindowRef.current = false; const attempt = attemptRef.current + 1; attemptRef.current = attempt; const ownsAttempt = () => attemptRef.current === attempt; @@ -115,7 +194,18 @@ export function ServerUpdateAction({ clearTimeout(expiry); expiry = armExpiry(); } + // The install can consume most of the request window, so the window + // armed at click time may already have lapsed by the time the restart + // actually begins. Re-arm from the handoff, not from the click. + expectServerRestart(environmentId); }; + // Armed before dispatch: the boot-service path acknowledges and then + // drops every connection, and even the respawn acknowledgement can lose + // the race against the disconnect. An explicit RPC failure proves the + // server stayed alive, so the expectation is withdrawn there; on + // success or interrupt it is re-armed from that point and expires on its + // own if the server never returns. + expectServerRestart(environmentId); void Promise.resolve() .then(() => updateServer({ @@ -129,14 +219,21 @@ export function ServerUpdateAction({ // An interrupt may be the expected boot-service disconnect, but it // can also be client-side cancellation before restart was accepted. // Release the action quietly; version sync will remove it when a - // successful replacement reconnects. + // successful replacement reconnects. Re-arm from here for the same + // reason as the success path: a long install may have outlasted the + // window armed at click time. if (isAtomCommandInterrupted(result)) { + expectServerRestart(environmentId); return; } + const failure = squashAtomCommandFailure(result); + if (!isUpdateAlreadyRunning(failure)) { + clearExpectedServerRestart(environmentId); + } toastManager.add({ type: "error", title: "Server update failed", - description: updateFailureMessage(squashAtomCommandFailure(result)), + description: updateFailureMessage(failure), }); return; } @@ -152,6 +249,9 @@ export function ServerUpdateAction({ }) .catch((error: unknown) => { if (!ownsAttempt()) return; + if (!isUpdateAlreadyRunning(error)) { + clearExpectedServerRestart(environmentId); + } toastManager.add({ type: "error", title: "Server update failed", diff --git a/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts b/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts index 654c2462a6f..bdc867b35dd 100644 --- a/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts +++ b/apps/web/src/components/cloud/cloudEnvironmentConnectionPresentation.test.ts @@ -7,7 +7,7 @@ function connection( phase: EnvironmentConnectionPresentation["phase"], error: string | null = null, ): EnvironmentConnectionPresentation { - return { phase, error, traceId: null }; + return { phase, error, reason: null, traceId: null, retryAt: null }; } describe("saved cloud environment connection presentation", () => { diff --git a/apps/web/src/serverRestartStore.test.ts b/apps/web/src/serverRestartStore.test.ts new file mode 100644 index 00000000000..53ed4b2fe06 --- /dev/null +++ b/apps/web/src/serverRestartStore.test.ts @@ -0,0 +1,134 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import { + SERVER_RESTART_EXPECTED_WINDOW_MS, + clearExpectedServerRestart, + expectServerRestart, + isExpectedRestartResolved, + isServerRestarting, + useServerRestartStore, +} from "./serverRestartStore"; + +const ENVIRONMENT_ID = "env-test" as EnvironmentId; + +function expiresAtFor(environmentId: EnvironmentId) { + return useServerRestartStore.getState().expiresAtByEnvironmentId[environmentId] ?? null; +} + +describe("serverRestartStore", () => { + beforeEach(() => { + vi.useFakeTimers(); + useServerRestartStore.setState({ expiresAtByEnvironmentId: {} }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("arms a window that expires on its own", () => { + expectServerRestart(ENVIRONMENT_ID); + expect(expiresAtFor(ENVIRONMENT_ID)).toBe(Date.now() + SERVER_RESTART_EXPECTED_WINDOW_MS); + }); + + it("extends the window when a slow install acknowledges late", () => { + expectServerRestart(ENVIRONMENT_ID); + vi.advanceTimersByTime(60_000); + expectServerRestart(ENVIRONMENT_ID); + + // Re-arming from the acknowledgement is what keeps a restart that begins + // minutes after the click from being presented as an outage. + expect(expiresAtFor(ENVIRONMENT_ID)).toBe(Date.now() + SERVER_RESTART_EXPECTED_WINDOW_MS); + }); + + it("clears only the requested environment", () => { + const other = "env-other" as EnvironmentId; + expectServerRestart(ENVIRONMENT_ID); + expectServerRestart(other); + + clearExpectedServerRestart(ENVIRONMENT_ID); + + expect(expiresAtFor(ENVIRONMENT_ID)).toBeNull(); + expect(expiresAtFor(other)).not.toBeNull(); + }); + + describe("isServerRestarting", () => { + it("does not claim a still-connected environment is restarting", () => { + // The window is armed before dispatch, while the old server is still + // connected. Gating on availability is what keeps that from showing the + // restarting banner the instant the user clicks — and what makes a + // window outliving its restart harmless. + expect(isServerRestarting({ restartExpected: true, environmentUnavailable: false })).toBe( + false, + ); + }); + + it("presents an armed window with an away environment as restarting", () => { + expect(isServerRestarting({ restartExpected: true, environmentUnavailable: true })).toBe( + true, + ); + }); + + it("falls back to the outage UI once the window lapses", () => { + // The server never came back; the calm banner must give way. + expect(isServerRestarting({ restartExpected: false, environmentUnavailable: true })).toBe( + false, + ); + }); + + it("does not dress up an unrelated outage as a restart once cleared", () => { + // The consumer clears the window after seeing the restart through, so a + // later ordinary outage inside the original 90s must read as an outage. + expectServerRestart(ENVIRONMENT_ID); + clearExpectedServerRestart(ENVIRONMENT_ID); + vi.advanceTimersByTime(10_000); + + expect(expiresAtFor(ENVIRONMENT_ID)).toBeNull(); + expect( + isServerRestarting({ + restartExpected: expiresAtFor(ENVIRONMENT_ID) !== null, + environmentUnavailable: true, + }), + ).toBe(false); + }); + }); + + describe("isExpectedRestartResolved", () => { + const base = { + sawRestartWindow: true, + restartExpected: true, + wentAway: false, + environmentUnavailable: false, + }; + + it("does not resolve while the environment is still away", () => { + expect( + isExpectedRestartResolved({ ...base, wentAway: true, environmentUnavailable: true }), + ).toBe(false); + }); + + it("resolves once the environment comes back after going away", () => { + // Something restarted but the skew is still here, so whatever came back + // did not carry the new version. + expect(isExpectedRestartResolved({ ...base, wentAway: true })).toBe(true); + }); + + it("resolves when the window lapses without the connection ever dropping", () => { + // systemd refused the restart and rolled back: the RPC already returned + // success, so nothing else can release the action. + expect(isExpectedRestartResolved({ ...base, restartExpected: false })).toBe(true); + }); + + it("does not resolve while the restart is still expected", () => { + expect(isExpectedRestartResolved(base)).toBe(false); + }); + + it("does not resolve before a window has been observed", () => { + // Guards the render between the click and the store update landing, + // which would otherwise look like an already-lapsed window. + expect( + isExpectedRestartResolved({ ...base, sawRestartWindow: false, restartExpected: false }), + ).toBe(false); + }); + }); +}); diff --git a/apps/web/src/serverRestartStore.ts b/apps/web/src/serverRestartStore.ts new file mode 100644 index 00000000000..4c294dc6ae6 --- /dev/null +++ b/apps/web/src/serverRestartStore.ts @@ -0,0 +1,138 @@ +/** + * Environment-scoped "a server restart is expected" overlay. + * + * Set around a server self-update so the connection UI can tell an + * intentional restart apart from a genuine outage: while the window is + * active, disconnects present as a calm "restarting" notice and the client + * retries on a fast flat cadence instead of escalating backoff. This is + * deliberately not a connection-supervisor phase — the supervisor state + * machine stays transport-only, and this store is UI-side intent that + * expires on its own if the server never comes back. + * + * Ending the window is the subtle part. It is armed while the old server is + * still connected (the acknowledgement races its own disconnect), so a + * connected environment cannot by itself mean the restart is over — that is + * how an earlier revision made the whole feature inert. But never ending it + * is also wrong: an unrelated outage later in the window would be dressed up + * as an intentional restart. So the window ends when the environment has + * gone away and come back, i.e. the restart it was armed for has been seen. + * + * A blip during a long install can consume the window early; the + * acknowledgement re-arms it, which is what covers that case. + * + * Note this observation is only used for presentation. The update action's + * own release path deliberately does not depend on it — coupling the two is + * what produced a family of bugs where each consumer invalidated the other's + * evidence. + */ +import type { EnvironmentId } from "@t3tools/contracts"; +import { useEffect, useReducer } from "react"; +import { create } from "zustand"; + +/** + * How long after the last update signal (dispatch, RPC acknowledgement, or + * interrupted RPC) a disconnect is still attributed to the restart. Covers + * the respawn shim (~3s), boot-service systemd swap, server boot, and relay + * re-registration with generous margin; after it lapses the normal error + * presentation returns. + */ +export const SERVER_RESTART_EXPECTED_WINDOW_MS = 90_000; + +interface ServerRestartState { + readonly expiresAtByEnvironmentId: Readonly>; + readonly expect: (environmentId: EnvironmentId) => void; + readonly clear: (environmentId: EnvironmentId) => void; +} + +export const useServerRestartStore = create()((set) => ({ + expiresAtByEnvironmentId: {}, + expect: (environmentId) => + set((state) => ({ + expiresAtByEnvironmentId: { + ...state.expiresAtByEnvironmentId, + [environmentId]: Date.now() + SERVER_RESTART_EXPECTED_WINDOW_MS, + }, + })), + clear: (environmentId) => + set((state) => { + if (!(environmentId in state.expiresAtByEnvironmentId)) { + return state; + } + const next = { ...state.expiresAtByEnvironmentId }; + delete next[environmentId]; + return { expiresAtByEnvironmentId: next }; + }), +})); + +export function expectServerRestart(environmentId: EnvironmentId): void { + useServerRestartStore.getState().expect(environmentId); +} + +export function clearExpectedServerRestart(environmentId: EnvironmentId): void { + useServerRestartStore.getState().clear(environmentId); +} + +/** + * Whether an unexpired restart window is armed for the environment. + * Re-renders when the window is armed, cleared, or lapses. + */ +export function useServerRestartExpected(environmentId: EnvironmentId | null): boolean { + const expiresAt = useServerRestartStore((state) => + environmentId === null ? null : (state.expiresAtByEnvironmentId[environmentId] ?? null), + ); + const [, tick] = useReducer((count: number) => count + 1, 0); + useEffect(() => { + if (expiresAt === null) { + return; + } + const remaining = expiresAt - Date.now(); + if (remaining <= 0) { + return; + } + const timer = setTimeout(tick, remaining + 50); + return () => clearTimeout(timer); + }, [expiresAt]); + return expiresAt !== null && expiresAt > Date.now(); +} + +/** + * Whether the environment should be presented as restarting rather than as an + * outage. Requires the environment to actually be away, which is why a window + * that outlives its restart is harmless: a reconnected environment reports + * false regardless of the window. + */ +export function isServerRestarting(input: { + readonly restartExpected: boolean; + readonly environmentUnavailable: boolean; +}): boolean { + return input.restartExpected && input.environmentUnavailable; +} + +/** + * Whether an expected restart has resolved without delivering a new version, + * so UI held for it should be released. + * + * The update RPC is acknowledged before the restart runs, so a rejected + * restart cannot report itself through the call. Two shapes resolve it, + * because a rejected restart may or may not have dropped the connection: + * the environment went away and came back (something restarted, but the skew + * is still here), or the window lapsed while the connection never dropped + * (no restart ever happened). + */ +export function isExpectedRestartResolved(input: { + /** A restart window was observed as armed at some point. */ + readonly sawRestartWindow: boolean; + /** A restart window is armed right now. */ + readonly restartExpected: boolean; + /** The environment has been unavailable at some point since arming. */ + readonly wentAway: boolean; + readonly environmentUnavailable: boolean; +}): boolean { + if (input.environmentUnavailable) { + return false; + } + if (input.wentAway) { + return true; + } + return input.sawRestartWindow && !input.restartExpected; +} diff --git a/packages/client-runtime/src/authorization/layer.test.ts b/packages/client-runtime/src/authorization/layer.test.ts index 1d2c6c6cca7..13531609d63 100644 --- a/packages/client-runtime/src/authorization/layer.test.ts +++ b/packages/client-runtime/src/authorization/layer.test.ts @@ -316,6 +316,42 @@ describe("RemoteEnvironmentAuthorization", () => { }), ); + it.effect("keeps a cached token when the endpoint is merely unreachable", () => + Effect.gen(function* () { + const cached = new TokenStore.RemoteDpopAccessToken({ + environmentId: ENVIRONMENT_ID, + label: DESCRIPTOR.label, + endpoint: ENDPOINT, + accessToken: "cached-access-token", + expiresAtEpochMs: Number.MAX_SAFE_INTEGER, + dpopThumbprint: "thumbprint-1", + }); + // A restarting server refuses connections outright rather than + // answering; re-bootstrapping would not produce a better endpoint, so + // the token must survive to keep reconnects to a single request. + const harness = yield* makeHarness({ + initialToken: cached, + responses: [], + }); + + yield* Effect.gen(function* () { + const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization; + for (let attempt = 0; attempt < 3; attempt++) { + const failure = yield* remote + .authorizeDpop({ + expectedEnvironmentId: ENVIRONMENT_ID, + obtainBootstrap: harness.obtainBootstrap, + }) + .pipe(Effect.flip); + expect(failure._tag).toBe("ConnectionTransientError"); + } + }).pipe(Effect.provide(harness.layer)); + + expect((yield* Ref.get(harness.tokens)).get(ENVIRONMENT_ID)).toBe(cached); + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(0); + }), + ); + it.effect("does not persist a refreshed token until its websocket ticket succeeds", () => Effect.gen(function* () { const harness = yield* makeHarness({ diff --git a/packages/client-runtime/src/authorization/service.ts b/packages/client-runtime/src/authorization/service.ts index 624ecf7f672..29fea911278 100644 --- a/packages/client-runtime/src/authorization/service.ts +++ b/packages/client-runtime/src/authorization/service.ts @@ -60,6 +60,24 @@ export class RemoteEnvironmentAuthorization extends Context.Service< const TOKEN_EXPIRY_SAFETY_MARGIN_MS = 60_000; const CACHED_ENDPOINT_FAILURE_THRESHOLD = 2; +/** + * Transient reasons that suggest the *cached endpoint itself* is stale — + * something answered and rejected or could not route the request — so + * re-bootstrapping may hand back a different endpoint. + * + * `network` and `timeout` are excluded on purpose: they only prove the + * endpoint was unreachable right now, which is the normal shape of a server + * restart or a brief outage. Evicting on those threw away a perfectly good + * token and forced a full relay bootstrap + token exchange on every + * reconnect, adding latency exactly when the server was coming back. + */ +const CACHED_ENDPOINT_STALE_REASONS: ReadonlySet = new Set([ + "endpoint-unavailable", + "remote-unavailable", + "relay-unavailable", + "transport", +]); + function mapDpopSocketError(error: RemoteEnvironmentAuthError | ConnectionAttemptError) { return error._tag === "ConnectionTransientError" || error._tag === "ConnectionBlockedError" ? error @@ -215,6 +233,11 @@ export const make = Effect.gen(function* () { } const mappedFailure = mapDpopSocketError(cachedSocket.failure); if (mappedFailure._tag === "ConnectionTransientError") { + if (!CACHED_ENDPOINT_STALE_REASONS.has(mappedFailure.reason)) { + // Pure unreachability: keep the cached token so the next attempt + // is a single ticket request rather than a full re-bootstrap. + return yield* mappedFailure; + } const failureCount = yield* recordCachedEndpointFailure(input.expectedEnvironmentId); if (failureCount < CACHED_ENDPOINT_FAILURE_THRESHOLD) { return yield* mappedFailure; diff --git a/packages/client-runtime/src/connection/presentation.test.ts b/packages/client-runtime/src/connection/presentation.test.ts index e13638a2b41..332fc3053cf 100644 --- a/packages/client-runtime/src/connection/presentation.test.ts +++ b/packages/client-runtime/src/connection/presentation.test.ts @@ -5,6 +5,7 @@ import * as Option from "effect/Option"; import { BearerConnectionProfile, type ConnectionCatalogEntry } from "./catalog.ts"; import { BearerConnectionTarget, + ConnectionBlockedError, ConnectionTransientError, type SupervisorConnectionState, } from "./model.ts"; @@ -59,7 +60,9 @@ describe("connection presentation", () => { expect(presentConnectionState(supervisorState({ phase: "connecting", attempt: 1 }))).toEqual({ phase: "connecting", error: null, + reason: null, traceId: null, + retryAt: null, }); expect( presentConnectionState( @@ -75,8 +78,10 @@ describe("connection presentation", () => { ), ).toEqual({ phase: "reconnecting", - error: "Socket closed.", + error: "The connection to the environment was interrupted.", + reason: "transport", traceId: "trace-previous", + retryAt: null, }); expect( presentConnectionState( @@ -93,8 +98,48 @@ describe("connection presentation", () => { ), ).toEqual({ phase: "reconnecting", - error: "Disconnected.", + error: "The connection to the environment was interrupted.", + reason: "transport", traceId: "trace-1", + retryAt: 1, + }); + }); + + it("summarizes transport failures without exposing endpoint internals", () => { + const presented = presentConnectionState( + supervisorState({ + phase: "backoff", + attempt: 2, + retryAt: 5, + lastFailure: new ConnectionTransientError({ + reason: "network", + detail: + "Failed to fetch remote environment endpoint https://prod-6e7fc14fddd8f0fb.t3coderelay.com/api/auth/websocket-ticket (HttpClientError: Transport error).", + }), + }), + ); + expect(presented.error).toBe("The environment could not be reached."); + expect(presented.error).not.toContain("t3coderelay.com"); + expect(presented.reason).toBe("network"); + }); + + it("keeps authored blocked-failure messages, which are already actionable", () => { + expect( + presentConnectionState( + supervisorState({ + phase: "blocked", + lastFailure: new ConnectionBlockedError({ + reason: "authentication", + detail: "The environment credential is invalid.", + }), + }), + ), + ).toEqual({ + phase: "error", + error: "The environment credential is invalid.", + reason: "authentication", + traceId: null, + retryAt: null, }); }); @@ -106,7 +151,7 @@ describe("connection presentation", () => { stage: "opening", attempt: 2, lastFailure: new ConnectionTransientError({ - reason: "transport", + reason: "timeout", detail: "Relay connection timed out.", traceId: "trace-retry", }), @@ -114,8 +159,10 @@ describe("connection presentation", () => { ), ).toEqual({ phase: "reconnecting", - error: "Relay connection timed out.", + error: "The environment did not respond in time.", + reason: "timeout", traceId: "trace-retry", + retryAt: null, }); }); @@ -127,7 +174,9 @@ describe("connection presentation", () => { const connection = { phase: "reconnecting", error: "Relay request timed out.", + reason: "timeout", traceId: "trace-retry", + retryAt: null, } as const; expect(connectionStatusText(connection)).toBe( "Failed to connect. Reconnecting... Reason: Relay request timed out.", @@ -147,7 +196,9 @@ describe("connection presentation", () => { ).toEqual({ phase: "offline", error: null, + reason: null, traceId: null, + retryAt: null, }); }); @@ -163,7 +214,9 @@ describe("connection presentation", () => { ).toEqual({ phase: "connected", error: null, + reason: null, traceId: null, + retryAt: null, }); }); @@ -181,7 +234,9 @@ describe("connection presentation", () => { ).toEqual({ phase: "available", error: null, + reason: null, traceId: null, + retryAt: null, }); }); }); diff --git a/packages/client-runtime/src/connection/presentation.ts b/packages/client-runtime/src/connection/presentation.ts index 168443deceb..dfd511581da 100644 --- a/packages/client-runtime/src/connection/presentation.ts +++ b/packages/client-runtime/src/connection/presentation.ts @@ -2,7 +2,7 @@ import type { ServerConfig } from "@t3tools/contracts"; import * as Option from "effect/Option"; import type { ConnectionCatalogEntry } from "./catalog.ts"; -import type { NetworkStatus, SupervisorConnectionState } from "./model.ts"; +import type { ConnectionAttemptError, NetworkStatus, SupervisorConnectionState } from "./model.ts"; export type EnvironmentConnectionPhase = | "available" @@ -12,10 +12,18 @@ export type EnvironmentConnectionPhase = | "connected" | "error"; +export type EnvironmentConnectionFailureReason = ConnectionAttemptError["reason"]; + export interface EnvironmentConnectionPresentation { readonly phase: EnvironmentConnectionPhase; readonly error: string | null; + readonly reason: EnvironmentConnectionFailureReason | null; readonly traceId: string | null; + /** Epoch ms of the next scheduled attempt while the supervisor sleeps in + backoff; null while an attempt is live. Callers that want to retry + early (retryNow) should only do so while this is set — waking an + in-flight attempt cancels and restarts it. */ + readonly retryAt: number | null; } export interface EnvironmentPresentation { @@ -24,38 +32,91 @@ export interface EnvironmentPresentation { readonly serverConfig: ServerConfig | null; } +/** + * Transport errors carry internal endpoint URLs and raw causes (see + * rpc/http.ts) that belong in diagnostics, not user-facing banners. Present + * a stable per-reason summary instead; the raw message stays available on + * the supervisor state for connection diagnostics. + */ +export function connectionFailureSummary(failure: ConnectionAttemptError): string { + switch (failure.reason) { + case "network": + return "The environment could not be reached."; + case "transport": + return "The connection to the environment was interrupted."; + case "timeout": + return "The environment did not respond in time."; + case "endpoint-unavailable": + case "remote-unavailable": + return "The environment is not accepting connections yet."; + case "relay-unavailable": + return "The relay is temporarily unavailable."; + // Blocked reasons are actionable, and their details are authored + // messages (errors.ts) rather than raw transport dumps. + case "authentication": + case "permission": + case "configuration": + case "unsupported": + return failure.message; + } +} + +function presentFailure(failure: ConnectionAttemptError | null): { + readonly error: string | null; + readonly reason: EnvironmentConnectionFailureReason | null; + readonly traceId: string | null; +} { + if (failure === null) { + return { error: null, reason: null, traceId: null }; + } + return { + error: connectionFailureSummary(failure), + reason: failure.reason, + traceId: failure.traceId ?? null, + }; +} + export function presentConnectionState( state: SupervisorConnectionState, ): EnvironmentConnectionPresentation { switch (state.phase) { case "available": - return { phase: "available", error: null, traceId: null }; + return { phase: "available", ...presentFailure(null), retryAt: null }; case "offline": - return { phase: "offline", error: null, traceId: null }; + return { phase: "offline", ...presentFailure(null), retryAt: null }; case "connecting": return { phase: state.attempt <= 1 && state.lastFailure === null ? "connecting" : "reconnecting", - error: state.lastFailure?.message ?? null, - traceId: state.lastFailure?.traceId ?? null, + ...presentFailure(state.lastFailure), + retryAt: null, }; case "connected": - return { phase: "connected", error: null, traceId: null }; + return { phase: "connected", ...presentFailure(null), retryAt: null }; case "backoff": return { phase: "reconnecting", - error: state.lastFailure?.message ?? null, - traceId: state.lastFailure?.traceId ?? null, + ...presentFailure(state.lastFailure), + retryAt: state.retryAt, }; case "blocked": return { phase: "error", - error: state.lastFailure?.message ?? null, - traceId: state.lastFailure?.traceId ?? null, + ...presentFailure(state.lastFailure), + retryAt: null, }; } } -export function connectionStatusText(connection: EnvironmentConnectionPresentation): string { +/** The status helpers only read phase and error, so callers holding a phase + and a message do not have to synthesize a whole presentation. traceId is + accepted (and ignored) because callers commonly carry it alongside. */ +export interface ConnectionStatusTextInput { + readonly phase: EnvironmentConnectionPhase; + readonly error: string | null; + readonly traceId?: string | null; +} + +export function connectionStatusText(connection: ConnectionStatusTextInput): string { switch (connection.phase) { case "available": return "Available"; @@ -76,7 +137,7 @@ export function connectionStatusText(connection: EnvironmentConnectionPresentati } } -export function connectionStatusTitle(connection: EnvironmentConnectionPresentation): string { +export function connectionStatusTitle(connection: ConnectionStatusTextInput): string { if (connection.phase === "reconnecting" && connection.error) { return "Failed to connect. Reconnecting..."; } diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 69699c7a839..49ac828f4ff 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -590,6 +590,12 @@ export const ServerSelfUpdateResult = Schema.Struct({ }); export type ServerSelfUpdateResult = typeof ServerSelfUpdateResult.Type; +/** Rejection reason when an update is dispatched while one is already + installing. Shared so clients can tell "nothing is happening" apart from + "a restart is already pending from an earlier request", which the update + UI must not treat as the end of the flow. */ +export const SERVER_SELF_UPDATE_ALREADY_RUNNING_REASON = "A server update is already in progress."; + export class ServerSelfUpdateError extends Schema.TaggedErrorClass()( "ServerSelfUpdateError", { @@ -600,4 +606,9 @@ export class ServerSelfUpdateError extends Schema.TaggedErrorClass