Fix rollout health-check timeouts from transient container states

agent/internal/agent/reporting.go

@@ -89,11 +89,26 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport
 			if a.ShouldSuppressServerlessContainerReport(c.DeploymentID) {
 				continue
 			}
-
-			status := "stopped"
-			if c.State == "running" {
+			// Intermediate podman states (e.g. "created" mid-deploy) must not be
+			// reported as stopped — the control plane would move the deployment
+			// into a stopped phase — nor omitted, which would read as the
+			// container being gone. They are reported as "transient" so the
+			// control plane keeps tracking the deployment without acting until
+			// the state settles. Settled non-running states ("stopped",
+			// "paused") map to stopped so the deployment leaves routing and
+			// drift reconciliation can repair it; the same goes for "unknown"
+			// or unrecognized states, since presence-only reporting there would
+			// leave a broken container marked healthy indefinitely.
+			var status string
+			switch c.State {
+			case "running":
 				status = "running"
-			} else if c.State == "exited" {
+			case "exited", "stopped", "paused":
+				status = "stopped"
+			case "created", "configured", "initialized", "stopping", "removing":
+				status = "transient"
+			default:
+				log.Printf("[status] container %s in unexpected state %q, reporting as stopped", c.ID, c.State)
 				status = "stopped"
 			}
 

web/lib/agent-status.ts

@@ -37,7 +37,7 @@ import { enqueueWork } from "@/lib/work-queue";
 type ContainerStatus = {
 	deploymentId: string;
 	containerId: string;
-	status: "running" | "stopped" | "failed";
+	status: "running" | "stopped" | "failed" | "transient";
 	healthStatus: "none" | "starting" | "healthy" | "unhealthy";
 };
 
@@ -89,7 +89,7 @@ export function getStoppedContainerReportUpdate(deployment: {
 	};
 }
 
-export function getStaleStoppedServerlessReportUpdate({
+export function getStaleStoppedReportUpdate({
 	hasHealthCheck,
 	healthStatus,
 }: {
@@ -779,6 +779,14 @@ export async function applyStatusReport(
 	}
 
 	for (const container of report.containers) {
+		// Transient containers (e.g. podman "created" mid-deploy) are reported
+		// for presence only — counted in reportedDeploymentIds above so the
+		// deployment isn't marked unknown or deleted, but their unsettled state
+		// must not drive any phase or health transition.
+		if (container.status === "transient") {
+			continue;
+		}
+
 		const healthStatus = container.healthStatus;
 
 		let [deployment] = container.deploymentId
@@ -875,6 +883,7 @@ export async function applyStatusReport(
 		let autohealRestartPayload: Record<string, unknown> | null = null;
 		let autohealRecreatePayload: Record<string, unknown> | null = null;
 		let autohealFailed = false;
+		let restoredToReady = false;
 
 		if (deployment.containerId !== container.containerId) {
 			updateFields.containerId = container.containerId;
@@ -921,16 +930,19 @@ export async function applyStatusReport(
 				.where(eq(serviceRevisions.id, deployment.serviceRevisionId))
 				.then((r) => r[0]);
 
-			if (revision?.specification.serverless.enabled) {
+			if (revision) {
 				Object.assign(
 					updateFields,
-					getStaleStoppedServerlessReportUpdate({
+					getStaleStoppedReportUpdate({
 						hasHealthCheck: revision.specification.healthCheck != null,
 						healthStatus,
 					}),
 				);
+				restoredToReady =
+					updateFields.observedPhase === "healthy" ||
+					updateFields.observedPhase === "running";
 				console.log(
-					`[health:restore] serverless deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`,
+					`[health:restore] deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`,
 				);
 			}
 		}
@@ -1009,6 +1021,7 @@ export async function applyStatusReport(
 					? "running"
 					: "starting";
 			updateFields.observedPhase = newStatus;
+			restoredToReady = newStatus === "running";
 			console.log(
 				`[health:restore] deployment ${deployment.id} restored from unknown to ${newStatus}`,
 			);
@@ -1079,6 +1092,24 @@ export async function applyStatusReport(
 			.set(updateFields)
 			.where(eq(deployments.id, deployment.id));
 
+		if (restoredToReady && deployment.rolloutId) {
+			const currentServerName = await getCurrentServerLogName();
+			await ingestRolloutLog(
+				deployment.rolloutId,
+				deployment.serviceId,
+				"health_check",
+				`Container is healthy on server ${currentServerName}`,
+			);
+			await inngest.send(
+				inngestEvents.resourceStatusChanged.create({
+					type: "deployment",
+					id: deployment.id,
+					parentType: "rollout",
+					parentId: deployment.rolloutId,
+				}),
+			);
+		}
+
 		if (autohealRestartPayload) {
 			await enqueueWork(serverId, "restart", autohealRestartPayload);
 		}

web/tests/agent-status.test.ts

@@ -49,10 +49,11 @@ vi.mock("@/lib/work-queue", () => ({
 import {
 	applyStatusReport,
 	getSleepTransitionDeploymentIds,
-	getStaleStoppedServerlessReportUpdate,
+	getStaleStoppedReportUpdate,
 	getStoppedContainerReportUpdate,
 	shouldAttachReportedContainer,
 } from "@/lib/agent-status";
+import { inngest } from "@/lib/inngest/client";
 
 beforeEach(() => {
 	mocks.selectResults.length = 0;
@@ -97,9 +98,9 @@ describe("agent status serverless attachment", () => {
 		});
 	});
 
-	it("restores stale stopped serverless observations from live running reports", () => {
+	it("restores stale stopped observations from live running reports", () => {
 		expect(
-			getStaleStoppedServerlessReportUpdate({
+			getStaleStoppedReportUpdate({
 				hasHealthCheck: false,
 				healthStatus: "none",
 			}),
@@ -110,7 +111,7 @@ describe("agent status serverless attachment", () => {
 		});
 
 		expect(
-			getStaleStoppedServerlessReportUpdate({
+			getStaleStoppedReportUpdate({
 				hasHealthCheck: true,
 				healthStatus: "starting",
 			}),
@@ -121,7 +122,7 @@ describe("agent status serverless attachment", () => {
 		});
 
 		expect(
-			getStaleStoppedServerlessReportUpdate({
+			getStaleStoppedReportUpdate({
 				hasHealthCheck: true,
 				healthStatus: "healthy",
 			}),
@@ -168,6 +169,34 @@ describe("agent status deployment cleanup", () => {
 		expect(mocks.db.delete).toHaveBeenCalledTimes(1);
 	});
 
+	it("retains a removed deployment whose container is reported in a transient state", async () => {
+		const deployment = {
+			id: "deployment_removed",
+			serviceId: "service_1",
+			serviceRevisionId: "revision_1",
+			serverId: "server_1",
+			containerId: "container_1",
+			runtimeDesiredState: "removed",
+			trafficState: "inactive",
+			observedPhase: "sleeping",
+			rolloutId: "rollout_1",
+		};
+		mocks.selectResults.push([deployment]);
+
+		await applyStatusReport("server_1", {
+			containers: [
+				{
+					deploymentId: deployment.id,
+					containerId: "container_1",
+					status: "transient",
+					healthStatus: "none",
+				},
+			],
+		});
+
+		expect(mocks.db.delete).not.toHaveBeenCalled();
+	});
+
 	it("retains a removed containerless deployment that reappears in the report", async () => {
 		const deployment = {
 			id: "deployment_removed",
@@ -196,3 +225,83 @@ describe("agent status deployment cleanup", () => {
 		expect(mocks.db.delete).not.toHaveBeenCalled();
 	});
 });
+
+describe("agent status stopped-phase recovery", () => {
+	it("promotes a non-serverless stopped deployment with a running container and notifies its rollout", async () => {
+		const deployment = {
+			id: "deployment_1",
+			serviceId: "service_1",
+			serviceRevisionId: "revision_1",
+			serverId: "server_1",
+			containerId: "container_1",
+			runtimeDesiredState: "running",
+			trafficState: "active",
+			observedPhase: "stopped",
+			rolloutId: "rollout_1",
+		};
+		mocks.selectResults.push(
+			[deployment],
+			[deployment],
+			[
+				{
+					specification: { serverless: { enabled: false }, healthCheck: null },
+				},
+			],
+		);
+
+		await applyStatusReport("server_1", {
+			containers: [
+				{
+					deploymentId: deployment.id,
+					containerId: "container_1",
+					status: "running",
+					healthStatus: "none",
+				},
+			],
+		});
+
+		expect(inngest.send).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "deployment",
+				id: "deployment_1",
+				parentType: "rollout",
+				parentId: "rollout_1",
+			}),
+		);
+	});
+
+	it("notifies the rollout when an unknown deployment is restored to running", async () => {
+		const deployment = {
+			id: "deployment_unknown",
+			serviceId: "service_1",
+			serviceRevisionId: "revision_1",
+			serverId: "server_1",
+			containerId: "container_1",
+			runtimeDesiredState: "running",
+			trafficState: "active",
+			observedPhase: "unknown",
+			rolloutId: "rollout_2",
+		};
+		mocks.selectResults.push([deployment], [deployment]);
+
+		await applyStatusReport("server_1", {
+			containers: [
+				{
+					deploymentId: deployment.id,
+					containerId: "container_1",
+					status: "running",
+					healthStatus: "none",
+				},
+			],
+		});
+
+		expect(inngest.send).toHaveBeenCalledWith(
+			expect.objectContaining({
+				type: "deployment",
+				id: "deployment_unknown",
+				parentType: "rollout",
+				parentId: "rollout_2",
+			}),
+		);
+	});
+});