diff --git a/agent/internal/agent/reporting.go b/agent/internal/agent/reporting.go index 9a3b5cb3..257bdf41 100644 --- a/agent/internal/agent/reporting.go +++ b/agent/internal/agent/reporting.go @@ -89,11 +89,26 @@ func (a *Agent) BuildStatusReport(includeResources bool) *agenthttp.StatusReport if a.ShouldSuppressServerlessContainerReport(c.DeploymentID) { continue } - - status := "stopped" - if c.State == "running" { + // Intermediate podman states (e.g. "created" mid-deploy) must not be + // reported as stopped — the control plane would move the deployment + // into a stopped phase — nor omitted, which would read as the + // container being gone. They are reported as "transient" so the + // control plane keeps tracking the deployment without acting until + // the state settles. Settled non-running states ("stopped", + // "paused") map to stopped so the deployment leaves routing and + // drift reconciliation can repair it; the same goes for "unknown" + // or unrecognized states, since presence-only reporting there would + // leave a broken container marked healthy indefinitely. + var status string + switch c.State { + case "running": status = "running" - } else if c.State == "exited" { + case "exited", "stopped", "paused": + status = "stopped" + case "created", "configured", "initialized", "stopping", "removing": + status = "transient" + default: + log.Printf("[status] container %s in unexpected state %q, reporting as stopped", c.ID, c.State) status = "stopped" } diff --git a/web/lib/agent-status.ts b/web/lib/agent-status.ts index d4f3f291..717f6225 100644 --- a/web/lib/agent-status.ts +++ b/web/lib/agent-status.ts @@ -37,7 +37,7 @@ import { enqueueWork } from "@/lib/work-queue"; type ContainerStatus = { deploymentId: string; containerId: string; - status: "running" | "stopped" | "failed"; + status: "running" | "stopped" | "failed" | "transient"; healthStatus: "none" | "starting" | "healthy" | "unhealthy"; }; @@ -89,7 +89,7 @@ export function getStoppedContainerReportUpdate(deployment: { }; } -export function getStaleStoppedServerlessReportUpdate({ +export function getStaleStoppedReportUpdate({ hasHealthCheck, healthStatus, }: { @@ -779,6 +779,14 @@ export async function applyStatusReport( } for (const container of report.containers) { + // Transient containers (e.g. podman "created" mid-deploy) are reported + // for presence only — counted in reportedDeploymentIds above so the + // deployment isn't marked unknown or deleted, but their unsettled state + // must not drive any phase or health transition. + if (container.status === "transient") { + continue; + } + const healthStatus = container.healthStatus; let [deployment] = container.deploymentId @@ -875,6 +883,7 @@ export async function applyStatusReport( let autohealRestartPayload: Record | null = null; let autohealRecreatePayload: Record | null = null; let autohealFailed = false; + let restoredToReady = false; if (deployment.containerId !== container.containerId) { updateFields.containerId = container.containerId; @@ -921,16 +930,19 @@ export async function applyStatusReport( .where(eq(serviceRevisions.id, deployment.serviceRevisionId)) .then((r) => r[0]); - if (revision?.specification.serverless.enabled) { + if (revision) { Object.assign( updateFields, - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: revision.specification.healthCheck != null, healthStatus, }), ); + restoredToReady = + updateFields.observedPhase === "healthy" || + updateFields.observedPhase === "running"; console.log( - `[health:restore] serverless deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`, + `[health:restore] deployment ${deployment.id} restored from ${deployment.observedPhase} to ${updateFields.observedPhase}`, ); } } @@ -1009,6 +1021,7 @@ export async function applyStatusReport( ? "running" : "starting"; updateFields.observedPhase = newStatus; + restoredToReady = newStatus === "running"; console.log( `[health:restore] deployment ${deployment.id} restored from unknown to ${newStatus}`, ); @@ -1079,6 +1092,24 @@ export async function applyStatusReport( .set(updateFields) .where(eq(deployments.id, deployment.id)); + if (restoredToReady && deployment.rolloutId) { + const currentServerName = await getCurrentServerLogName(); + await ingestRolloutLog( + deployment.rolloutId, + deployment.serviceId, + "health_check", + `Container is healthy on server ${currentServerName}`, + ); + await inngest.send( + inngestEvents.resourceStatusChanged.create({ + type: "deployment", + id: deployment.id, + parentType: "rollout", + parentId: deployment.rolloutId, + }), + ); + } + if (autohealRestartPayload) { await enqueueWork(serverId, "restart", autohealRestartPayload); } diff --git a/web/tests/agent-status.test.ts b/web/tests/agent-status.test.ts index a3fc860c..e7553218 100644 --- a/web/tests/agent-status.test.ts +++ b/web/tests/agent-status.test.ts @@ -49,10 +49,11 @@ vi.mock("@/lib/work-queue", () => ({ import { applyStatusReport, getSleepTransitionDeploymentIds, - getStaleStoppedServerlessReportUpdate, + getStaleStoppedReportUpdate, getStoppedContainerReportUpdate, shouldAttachReportedContainer, } from "@/lib/agent-status"; +import { inngest } from "@/lib/inngest/client"; beforeEach(() => { mocks.selectResults.length = 0; @@ -97,9 +98,9 @@ describe("agent status serverless attachment", () => { }); }); - it("restores stale stopped serverless observations from live running reports", () => { + it("restores stale stopped observations from live running reports", () => { expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: false, healthStatus: "none", }), @@ -110,7 +111,7 @@ describe("agent status serverless attachment", () => { }); expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: true, healthStatus: "starting", }), @@ -121,7 +122,7 @@ describe("agent status serverless attachment", () => { }); expect( - getStaleStoppedServerlessReportUpdate({ + getStaleStoppedReportUpdate({ hasHealthCheck: true, healthStatus: "healthy", }), @@ -168,6 +169,34 @@ describe("agent status deployment cleanup", () => { expect(mocks.db.delete).toHaveBeenCalledTimes(1); }); + it("retains a removed deployment whose container is reported in a transient state", async () => { + const deployment = { + id: "deployment_removed", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "removed", + trafficState: "inactive", + observedPhase: "sleeping", + rolloutId: "rollout_1", + }; + mocks.selectResults.push([deployment]); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "transient", + healthStatus: "none", + }, + ], + }); + + expect(mocks.db.delete).not.toHaveBeenCalled(); + }); + it("retains a removed containerless deployment that reappears in the report", async () => { const deployment = { id: "deployment_removed", @@ -196,3 +225,83 @@ describe("agent status deployment cleanup", () => { expect(mocks.db.delete).not.toHaveBeenCalled(); }); }); + +describe("agent status stopped-phase recovery", () => { + it("promotes a non-serverless stopped deployment with a running container and notifies its rollout", async () => { + const deployment = { + id: "deployment_1", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "stopped", + rolloutId: "rollout_1", + }; + mocks.selectResults.push( + [deployment], + [deployment], + [ + { + specification: { serverless: { enabled: false }, healthCheck: null }, + }, + ], + ); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "running", + healthStatus: "none", + }, + ], + }); + + expect(inngest.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: "deployment", + id: "deployment_1", + parentType: "rollout", + parentId: "rollout_1", + }), + ); + }); + + it("notifies the rollout when an unknown deployment is restored to running", async () => { + const deployment = { + id: "deployment_unknown", + serviceId: "service_1", + serviceRevisionId: "revision_1", + serverId: "server_1", + containerId: "container_1", + runtimeDesiredState: "running", + trafficState: "active", + observedPhase: "unknown", + rolloutId: "rollout_2", + }; + mocks.selectResults.push([deployment], [deployment]); + + await applyStatusReport("server_1", { + containers: [ + { + deploymentId: deployment.id, + containerId: "container_1", + status: "running", + healthStatus: "none", + }, + ], + }); + + expect(inngest.send).toHaveBeenCalledWith( + expect.objectContaining({ + type: "deployment", + id: "deployment_unknown", + parentType: "rollout", + parentId: "rollout_2", + }), + ); + }); +});