diff --git a/components/ws-manager-bridge/src/bridge.ts b/components/ws-manager-bridge/src/bridge.ts index 45272429b7ac3e..2f8051580baff5 100644 --- a/components/ws-manager-bridge/src/bridge.ts +++ b/components/ws-manager-bridge/src/bridge.ts @@ -446,13 +446,13 @@ export class WorkspaceManagerBridge implements Disposable { const installation = this.cluster.name; log.debug("Controlling instances...", { installation }); - const runningInstances = await this.workspaceDB + const nonStoppedInstances = await this.workspaceDB .trace(ctx) .findRunningInstancesWithWorkspaces(installation, undefined, true); // Control running workspace instances against ws-manager try { - await this.controlRunningInstances(ctx, runningInstances, clientProvider); + await this.controlNonStoppedWSManagerManagedInstances(ctx, nonStoppedInstances, clientProvider); disconnectStarted = Number.MAX_SAFE_INTEGER; // Reset disconnect period } catch (err) { @@ -466,7 +466,7 @@ export class WorkspaceManagerBridge implements Disposable { } // Control workspace instances against timeouts - await this.controlInstancesTimeouts(ctx, runningInstances); + await this.controlInstancesTimeouts(ctx, nonStoppedInstances); log.debug("Done controlling instances.", { installation }); } catch (err) { @@ -485,17 +485,17 @@ export class WorkspaceManagerBridge implements Disposable { * This methods controls all instances that we have currently marked as "running" in the DB. * It checks whether they are still running with their respective ws-manager, and if not, marks them as stopped in the DB. */ - protected async controlRunningInstances( + protected async controlNonStoppedWSManagerManagedInstances( parentCtx: TraceContext, runningInstances: RunningWorkspaceInfo[], clientProvider: ClientProvider, ) { const installation = this.config.installation; - const span = TraceContext.startSpan("controlRunningInstances", parentCtx); + const span = TraceContext.startSpan("controlNonStoppedWSManagerManagedInstances", parentCtx); const ctx = { span }; try { - log.debug("Controlling running instances...", { installation }); + log.debug("Controlling non-stopped instances that are managed by WS Manager...", { installation }); const runningInstancesIdx = new Map(); runningInstances.forEach((i) => runningInstancesIdx.set(i.latestInstance.id, i)); @@ -504,9 +504,27 @@ export class WorkspaceManagerBridge implements Disposable { const actuallyRunningInstances = await client.getWorkspaces(ctx, new GetWorkspacesRequest()); actuallyRunningInstances.getStatusList().forEach((s) => runningInstancesIdx.delete(s.getId())); + // runningInstancesIdx only contains instances that ws-manager is not aware of for (const [instanceId, ri] of runningInstancesIdx.entries()) { const instance = ri.latestInstance; - if (instance.status.phase !== "running") { + const phase = instance.status.phase; + if (phase !== "running") { + // This below if block is to validate the planned fix + if ( + phase === "pending" || + phase === "creating" || + phase === "initializing" || + (phase === "stopping" && + instance.stoppingTime && + durationLongerThanSeconds(Date.parse(instance.stoppingTime), 10)) + ) { + log.info("Logging to validate #12902. Should mark as stopped in database.", { + instanceId, + workspaceId: instance.workspaceId, + installation, + phase, + }); + } log.debug({ instanceId }, "Skipping instance", { phase: instance.status.phase, creationTime: instance.creationTime, @@ -516,9 +534,8 @@ export class WorkspaceManagerBridge implements Disposable { } log.info( - { instanceId, workspaceId: instance.workspaceId }, "Database says the instance is running, but wsman does not know about it. Marking as stopped in database.", - { installation }, + { instanceId, workspaceId: instance.workspaceId, installation, phase }, ); await this.markWorkspaceInstanceAsStopped(ctx, ri, new Date()); } @@ -603,6 +620,7 @@ export class WorkspaceManagerBridge implements Disposable { const nowISO = now.toISOString(); info.latestInstance.stoppingTime = nowISO; info.latestInstance.stoppedTime = nowISO; + info.latestInstance.status.message = `Stopped by ws-manager-bridge. Previously in phase ${info.latestInstance.status.phase}`; info.latestInstance.status.phase = "stopped"; await this.workspaceDB.trace(ctx).storeInstance(info.latestInstance);