(CODUX-220) Settle preparation failures and recovery

src/services/cli-workflow-service.ts

@@ -486,7 +486,7 @@ export class CliWorkflowService {
             dispatchStatus: "blocked",
             errorMessage: blocker,
             workerBranch: null,
-          });
+          }, ctx.executionInvocationId);
           this.appendExecutionEvent(args, "cli_workflow_blocked", {
             provider: args.provider,
             category,
@@ -507,7 +507,7 @@ export class CliWorkflowService {
           // branch, otherwise the orchestrator treats it as merge evidence and
           // falsely advances/merges the task.
           workerBranch: null,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_completed", {
           provider: args.provider,
           outcome: "no_changes",
@@ -534,7 +534,7 @@ export class CliWorkflowService {
         finishedAt,
         workerBranch: args.workerBranch,
         dispatchStatus: "completed",
-      });
+      }, ctx.executionInvocationId);
 
       const { prUrl } = await executePrFinalizeStage(ctx, { completionTimestamp: finishedAt });
       this.updateExecutionState(args, {
@@ -543,7 +543,7 @@ export class CliWorkflowService {
         prUrl,
         workerBranch: args.workerBranch,
         dispatchStatus: "completed",
-      });
+      }, ctx.executionInvocationId);
       this.appendExecutionEvent(args, "cli_pr_finalized", {
         provider: args.provider,
         prUrl: prUrl || null,
@@ -590,7 +590,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: "cancelled",
           errorMessage: "Workflow cancelled by dashboard control.",
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_cancel_requested", {
           provider: args.provider,
           sessionId: args.sessionId,
@@ -610,7 +610,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: workflowSettings.retryOnRateLimit ? "quota" : "failed",
           errorMessage: message,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_rate_limited", {
           provider: args.provider,
           errorMessage: message,
@@ -635,7 +635,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: "quota",
           errorMessage: message,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_quota", {
           provider: args.provider,
           errorMessage: message,
@@ -660,7 +660,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: "failed",
           errorMessage: message,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_failed", {
           provider: args.provider,
           errorMessage: message,
@@ -683,7 +683,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: "blocked",
           errorMessage: message,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_blocked", {
           provider: args.provider,
           category: isNonRecoverableGitWorkflowError(message) ? "git_configuration" : "execution_environment",
@@ -705,7 +705,7 @@ export class CliWorkflowService {
           finishedAt,
           dispatchStatus: "failed",
           errorMessage: message,
-        });
+        }, ctx.executionInvocationId);
         this.appendExecutionEvent(args, "cli_workflow_failed", {
           provider: args.provider,
           errorMessage: message,
@@ -716,11 +716,12 @@ export class CliWorkflowService {
           message,
         });
       }
+      const invocationStatus = abortController.signal.aborted ? "cancelled" : "failed";
       this.finalizeExecutionInvocation(
         ctx.executionInvocationId,
-        abortController.signal.aborted ? "cancelled" : "failed",
+        invocationStatus,
         finishedAt,
-        message,
+        invocationStatus === "cancelled" ? "Workflow cancelled by dashboard control." : message,
       );
     } finally {
       try {
@@ -865,12 +866,23 @@ export class CliWorkflowService {
       dispatchStatus: NonNullable<UpdateTaskDispatchInput["status"]>;
       errorMessage?: string;
     },
+    executionInvocationId?: string,
   ): void {
     const taskRun = this.resolveTaskRun(args);
     if (!taskRun || !this.deps.executionRepository) {
       return;
     }
 
+    // Cancellation is persisted before the active provider/workflow has
+    // necessarily observed its abort signal. Ignore a late pipeline update so
+    // the cancelled task and dispatch cannot drift back to another state.
+    if (
+      executionInvocationId
+      && this.deps.executionRepository.getExecutionInvocation(executionInvocationId)?.status === "cancelled"
+    ) {
+      return;
+    }
+
     if (this.isSprintRunCancelled(taskRun.sprintRunId)) {
       this.markTaskRunCancelledBySprintStop(taskRun, input.finishedAt, input.errorMessage ?? "Sprint run was cancelled.");
       return;
@@ -920,7 +932,23 @@ export class CliWorkflowService {
     this.deps.executionRepository.updateExecutionInvocation(invocationId, {
       status,
       finishedAt,
-      ...(status === "completed" ? { errorMessage: null } : { errorMessage: errorMessage ?? null }),
+      errorMessage: status === "completed" ? null : errorMessage ?? null,
+      lastErrorCategory: status === "failed" ? invocation.lastErrorCategory ?? "UNKNOWN" : null,
+      lastErrorMessage: status === "completed" ? null : errorMessage ?? null,
+      lastRetryAfterIso: null,
+    });
+    this.deps.executionRepository.appendExecutionInvocationMessage(invocationId, {
+      role: "system",
+      contentMarkdown: status === "completed"
+        ? "CLI workflow completed successfully."
+        : status === "cancelled"
+          ? `CLI workflow cancelled${errorMessage ? `: ${errorMessage}` : "."}`
+          : `CLI workflow failed${errorMessage ? `: ${errorMessage}` : "."}`,
+      metadata: {
+        kind: "cli_workflow_finalized",
+        status,
+      },
+      createdAt: finishedAt,
     });
   }
 

src/services/execution-invocation-control-service.ts

@@ -111,39 +111,51 @@ export class ExecutionInvocationControlService {
     const providerInvocation = invocation.providerInvocationId
       ? this.deps.executionRepository.getProviderInvocationUsage(invocation.providerInvocationId)
       : null;
-
-    await this.requestActiveDispatchStop(invocation, CANCEL_MESSAGE);
-    const stoppedContainerIds = await this.stopDockerContainers(invocation, providerInvocation);
     const finishedAt = new Date().toISOString();
 
-    if (providerInvocation?.status === "running") {
-      this.deps.executionRepository.updateProviderInvocationUsage(providerInvocation.id, {
-        status: "cancelled",
-        finishedAt,
-        durationMs: calculateDurationMs(providerInvocation.startedAt, finishedAt) ?? undefined,
-      });
-    }
-
-    this.closeTaskRuntimeForRetry(invocation, finishedAt);
-
+    // Close the durable invocation before asking the active workflow to stop.
+    // The abort rejection can race this control request, and workflow/provider
+    // finalizers intentionally refuse to replace an already-terminal row.
     this.deps.executionRepository.updateExecutionInvocation(invocation.id, {
       status: "cancelled",
       finishedAt,
       errorMessage: CANCEL_MESSAGE,
+      lastErrorCategory: null,
+      lastErrorMessage: CANCEL_MESSAGE,
+      lastRetryAfterIso: null,
     });
     this.deps.executionRepository.appendExecutionInvocationMessage(invocation.id, {
       role: "system",
-      contentMarkdown: stoppedContainerIds.length > 0
-        ? `${CANCEL_MESSAGE} Stopped Docker container${stoppedContainerIds.length === 1 ? "" : "s"} ${stoppedContainerIds.join(", ")}.`
-        : CANCEL_MESSAGE,
+      contentMarkdown: CANCEL_MESSAGE,
       metadata: {
         cancellation: "dashboard_invocation_cancel",
         providerInvocationId: providerInvocation?.id ?? null,
-        stoppedContainerIds,
       },
       createdAt: finishedAt,
     });
 
+    await this.requestActiveDispatchStop(invocation, CANCEL_MESSAGE);
+    const stoppedContainerIds = await this.stopDockerContainers(invocation, providerInvocation).catch((error: unknown) => {
+      this.deps.logger?.warn("Failed to stop all Docker containers for invocation cancellation", {
+        invocationId: invocation.id,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return [];
+    });
+
+    const activeProviderInvocation = providerInvocation
+      ? this.deps.executionRepository.getProviderInvocationUsage(providerInvocation.id)
+      : null;
+    if (activeProviderInvocation?.status === "running") {
+      this.deps.executionRepository.updateProviderInvocationUsage(activeProviderInvocation.id, {
+        status: "cancelled",
+        finishedAt,
+        durationMs: calculateDurationMs(activeProviderInvocation.startedAt, finishedAt) ?? undefined,
+      });
+    }
+
+    this.closeTaskRuntimeForRetry(invocation, finishedAt);
+
     return {
       cancelled: true,
       invocationId,

src/services/runtime-recovery/invocation-recovery.ts

@@ -177,6 +177,9 @@ export class InvocationRecoveryService {
         status: resolution.status,
         finishedAt: reconciledAt,
         errorMessage: resolution.status === "failed" ? resolution.message : null,
+        lastErrorCategory: resolution.status === "failed" ? invocation.lastErrorCategory ?? "UNKNOWN" : null,
+        lastErrorMessage: resolution.status === "failed" ? resolution.message : null,
+        lastRetryAfterIso: null,
       });
       this.deps.executionRepository.appendExecutionInvocationMessage(invocation.id, {
         role: "system",
@@ -230,11 +233,25 @@ export class InvocationRecoveryService {
     const sprintRun = invocation.sprintRunId ? this.deps.executionRepository.getSprintRun(invocation.sprintRunId) : null;
     if (sprintRun && ["completed", "failed", "cancelled"].includes(sprintRun.status)) {
       return {
-        status: "failed",
+        status: sprintRun.status === "cancelled" ? "cancelled" : "failed",
         message: `Recovered stale task coding invocation after the linked sprint run was already ${sprintRun.status}.`,
       };
     }
 
+    const dispatch = invocation.dispatchId ? this.deps.executionRepository.getTaskDispatch(invocation.dispatchId) : null;
+    if (
+      dispatch
+      && dispatch.status !== "paused"
+      && !ACTIVE_DISPATCH_STATUSES.includes(dispatch.status as (typeof ACTIVE_DISPATCH_STATUSES)[number])
+    ) {
+      return {
+        status: dispatch.status === "completed"
+          ? "completed"
+          : dispatch.status === "cancelled" ? "cancelled" : "failed",
+        message: `Recovered stale task coding invocation after the linked task dispatch was already ${dispatch.status}.`,
+      };
+    }
+
     const referenceAt = Date.parse(invocation.lastMessageAt || invocation.startedAt);
     const ageMs = Number.isFinite(referenceAt) ? Date.now() - referenceAt : 0;
 

src/services/runtime-startup-recovery-service.ts

@@ -134,10 +134,13 @@ export class RuntimeStartupRecoveryService {
     const reconciledStructuredInvocationIds = await invocationRecovery.reconcileInterruptedStructuredInvocations(activeContainerSessionIds);
     const rehydratedSprintRunIds = this.rehydrateDurableProviderSprintRuns();
     const restartPolicySyncedOrphanedSprintIds = this.syncOrphanedRunningSprintProjections();
-    const reconciledTaskCodingInvocationIds = await invocationRecovery.reconcileInterruptedTaskCodingInvocations(activeContainerSessionIds);
-    const reconciledTaskCodingProviderIds = invocationRecovery.reconcileOrphanedTaskCodingProviderInvocations();
     const reconciledTerminalProviderDispatchIds = this.reconcileTerminalProviderBackedDispatches();
     const reconciledTerminalDispatchIds = this.reconcileTerminalTaskRunDispatches();
+    // Settle task/dispatch truth before the workflow-level audit row. Provider
+    // completion alone does not mean a CLI workflow completed because Git and
+    // PR finalization happen after the provider exits.
+    const reconciledTaskCodingInvocationIds = await invocationRecovery.reconcileInterruptedTaskCodingInvocations(activeContainerSessionIds);
+    const reconciledTaskCodingProviderIds = invocationRecovery.reconcileOrphanedTaskCodingProviderInvocations();
     const reconciledDuplicateDispatchIds = this.reconcileDuplicateActiveTaskDispatches();
     const reconciledTaskRunIds = this.reconcileInterruptedTaskRuns();
     const reconciledPausedSprintRunIds = this.reconcileStalePausedSprintRuns();