diff --git a/docs/CONTROLLED_SESSION_DESIGN.md b/docs/CONTROLLED_SESSION_DESIGN.md index b1c944b..296fe96 100644 --- a/docs/CONTROLLED_SESSION_DESIGN.md +++ b/docs/CONTROLLED_SESSION_DESIGN.md @@ -1,6 +1,6 @@ --- status: Active -updated: 2026-08-09 +updated: 2026-08-10 summary: Capability-scoped execution sessions that inherit Reploy's global container sandbox. --- @@ -55,7 +55,13 @@ summary: Capability-scoped execution sessions that inherit Reploy's global conta completion and result acknowledgement, and removes both containers and the private channel. A workload that starts before a later startup step fails is still terminated and its output is finalized through the same barrier. - Crash watchdogs and restart reconciliation remain the next ownership phase; + Before creating any session resource, the planned controller, workload, and + private-channel ownership plus the session, lease, and boot identities are + now durably recorded in the existing live-run state. Reploy monotonically + fills each exact full container ID after Docker creates it, and both IDs are + durable before either process starts. Verified cleanup removes that record; + failed or unverifiable partial-preparation cleanup retains it. The watchdog + and restart reconciliation remain the next ownership phases, and controlled-session networking remains a later phase. - Initial runtime: Linux containers under Docker - Motivating clients: OmegaFlow recording, sandboxed AI agents, security diff --git a/internal/deploy/live_run_queue.go b/internal/deploy/live_run_queue.go index 14e7eb2..aa790aa 100644 --- a/internal/deploy/live_run_queue.go +++ b/internal/deploy/live_run_queue.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "regexp" "github.com/omry/reploy/internal/canonical" @@ -55,9 +56,28 @@ type LiveRunV1 struct { } type LiveRunQueueV1 struct { - Schema string `json:"schema"` - Runs []LiveRunV1 `json:"runs"` - Cleanup []LiveRunContainerCleanupV1 `json:"cleanup,omitempty"` + Schema string `json:"schema"` + Runs []LiveRunV1 `json:"runs"` + ControlledSessions []ControlledSessionOwnershipV1 `json:"controlled_sessions,omitempty"` + Cleanup []LiveRunContainerCleanupV1 `json:"cleanup,omitempty"` +} + +type ControlledSessionOwnershipV1 struct { + LiveRunID string `json:"live_run_id"` + BootSession string `json:"boot_session"` + SessionHandle string `json:"session_handle"` + ChannelDirectory string `json:"channel_directory"` + Controller ControlledSessionContainerOwnershipV1 `json:"controller"` + Workload ControlledSessionContainerOwnershipV1 `json:"workload"` +} + +type ControlledSessionContainerOwnershipV1 struct { + Role string `json:"role"` + ID string `json:"id"` + Name string `json:"name"` + DeploymentID string `json:"deployment_id"` + GenerationReference string `json:"generation_reference"` + BuildIdentity string `json:"build_identity"` } type LiveRunRecoveryReasonV1 string @@ -98,6 +118,9 @@ var ErrLiveRunConflict = errors.New("another run must finish first") var liveRunIDPatternV1 = regexp.MustCompile(`^run-[0-9a-f]{16}$`) var controlMarkerIDPatternV1 = regexp.MustCompile(`^control-[0-9a-f]{16}$`) +var controlledSessionHandlePatternV1 = regexp.MustCompile(`^session-[0-9a-f]{64}$`) +var controlledSessionContainerIDPatternV1 = regexp.MustCompile(`^[0-9a-f]{64}$`) +var controlledSessionBuildIdentityPatternV1 = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) func NewLiveRunQueueV1() LiveRunQueueV1 { return LiveRunQueueV1{Schema: LiveRunQueueSchemaV1, Runs: []LiveRunV1{}} @@ -189,6 +212,14 @@ func ValidateLiveRunQueueV1(queue LiveRunQueueV1) error { if queue.Runs == nil { return fmt.Errorf("live run queue runs must use an array") } + for index, ownership := range queue.ControlledSessions { + if err := validateControlledSessionOwnershipV1(ownership); err != nil { + return fmt.Errorf("live run queue controlled session %d: %w", index, err) + } + if index > 0 && queue.ControlledSessions[index-1].LiveRunID >= ownership.LiveRunID { + return fmt.Errorf("live run queue controlled sessions must be sorted and unique by live run ID") + } + } for index, cleanup := range queue.Cleanup { if err := validateLiveRunContainerCleanupV1(cleanup); err != nil { return fmt.Errorf("live run queue cleanup entry %d: %w", index, err) @@ -247,6 +278,65 @@ func ValidateLiveRunQueueV1(queue LiveRunQueueV1) error { return nil } +func validateControlledSessionOwnershipV1(ownership ControlledSessionOwnershipV1) error { + if err := ValidateLiveRunIDV1(ownership.LiveRunID); err != nil { + return fmt.Errorf("live run ID: %w", err) + } + if err := validateBootSessionIDV1(ownership.BootSession); err != nil { + return err + } + if !controlledSessionHandlePatternV1.MatchString(ownership.SessionHandle) { + return fmt.Errorf("session handle must use session- followed by 64 lowercase hexadecimal characters") + } + if !filepath.IsAbs(ownership.ChannelDirectory) || filepath.Clean(ownership.ChannelDirectory) != ownership.ChannelDirectory || !safeRecoveryIdentity(ownership.ChannelDirectory) { + return fmt.Errorf("channel directory must be a clean absolute path") + } + if err := validateControlledSessionContainerOwnershipStateV1(ownership.Controller, "controller"); err != nil { + return fmt.Errorf("controller: %w", err) + } + if err := validateControlledSessionContainerOwnershipStateV1(ownership.Workload, "workload"); err != nil { + return fmt.Errorf("workload: %w", err) + } + if ownership.Controller.ID == "" && ownership.Workload.ID != "" { + return fmt.Errorf("workload container ID cannot be recorded before the controller container ID") + } + if ownership.Controller.ID != "" && ownership.Workload.ID != "" && ownership.Controller.ID == ownership.Workload.ID { + return fmt.Errorf("controller and workload must name different containers") + } + return nil +} + +func validateControlledSessionContainerOwnershipV1(ownership ControlledSessionContainerOwnershipV1, role string) error { + if err := validateControlledSessionContainerOwnershipStateV1(ownership, role); err != nil { + return err + } + if ownership.ID == "" { + return fmt.Errorf("container ID must use 64 lowercase hexadecimal characters") + } + return nil +} + +func validateControlledSessionContainerOwnershipStateV1(ownership ControlledSessionContainerOwnershipV1, role string) error { + if ownership.Role != role { + return fmt.Errorf("role must be %q", role) + } + if ownership.ID != "" && !controlledSessionContainerIDPatternV1.MatchString(ownership.ID) { + return fmt.Errorf("container ID must use 64 lowercase hexadecimal characters") + } + for label, value := range map[string]string{ + "name": ownership.Name, "deployment ID": ownership.DeploymentID, + "generation reference": ownership.GenerationReference, + } { + if !safeRecoveryIdentity(value) { + return fmt.Errorf("%s must be nonempty safe text", label) + } + } + if !controlledSessionBuildIdentityPatternV1.MatchString(ownership.BuildIdentity) { + return fmt.Errorf("build identity must be a sha256 digest") + } + return nil +} + func validateLiveRunContainerCleanupV1(cleanup LiveRunContainerCleanupV1) error { if !safeRecoveryIdentity(cleanup.Container) { return fmt.Errorf("cleanup container must be nonempty safe text") @@ -506,14 +596,19 @@ func ControlMarkersV1(queue LiveRunQueueV1) []ControlMarkerV1 { } func cloneLiveRunQueueV1(queue LiveRunQueueV1) LiveRunQueueV1 { + var controlledSessions []ControlledSessionOwnershipV1 + if queue.ControlledSessions != nil { + controlledSessions = append([]ControlledSessionOwnershipV1{}, queue.ControlledSessions...) + } var cleanup []LiveRunContainerCleanupV1 if queue.Cleanup != nil { cleanup = append([]LiveRunContainerCleanupV1{}, queue.Cleanup...) } return LiveRunQueueV1{ - Schema: queue.Schema, - Runs: append([]LiveRunV1{}, queue.Runs...), - Cleanup: cleanup, + Schema: queue.Schema, + Runs: append([]LiveRunV1{}, queue.Runs...), + ControlledSessions: controlledSessions, + Cleanup: cleanup, } } diff --git a/internal/deploy/live_run_queue_file.go b/internal/deploy/live_run_queue_file.go index ee11810..9981a32 100644 --- a/internal/deploy/live_run_queue_file.go +++ b/internal/deploy/live_run_queue_file.go @@ -128,6 +128,155 @@ func (lock *OperationLock) RecordLiveRunContainerV1(id string, container string) return fmt.Errorf("live run %q is not outstanding", id) } +// RecordControlledSessionOwnershipV1 durably binds the planned resources to an +// active admitted shell and monotonically fills each exact container ID after +// Docker returns it. The boot identity comes from the admitted run already +// protected by this lock. +func (lock *OperationLock) RecordControlledSessionOwnershipV1(ownership ControlledSessionOwnershipV1) (ControlledSessionOwnershipV1, error) { + if lock == nil { + return ControlledSessionOwnershipV1{}, fmt.Errorf("record controlled session ownership requires an operation lock") + } + if err := ValidateLiveRunIDV1(ownership.LiveRunID); err != nil { + return ControlledSessionOwnershipV1{}, err + } + lock.mutex.Lock() + defer lock.mutex.Unlock() + path, err := lock.liveRunQueuePathLockedV1() + if err != nil { + return ControlledSessionOwnershipV1{}, err + } + queue, _, err := readLiveRunQueuePathV1(path) + if err != nil { + return ControlledSessionOwnershipV1{}, err + } + var admitted *LiveRunV1 + for index := range queue.Runs { + if queue.Runs[index].ID == ownership.LiveRunID { + admitted = &queue.Runs[index] + break + } + } + if admitted == nil { + return ControlledSessionOwnershipV1{}, fmt.Errorf("live run %q is not outstanding", ownership.LiveRunID) + } + if admitted.Status != LiveRunStatusActiveV1 || admitted.Kind != LiveRunKindShellV1 { + return ControlledSessionOwnershipV1{}, fmt.Errorf("controlled session live run %q must be an active shell", ownership.LiveRunID) + } + if admitted.Container != "" { + return ControlledSessionOwnershipV1{}, fmt.Errorf("controlled session live run %q already names container %q", ownership.LiveRunID, admitted.Container) + } + if admitted.GenerationReference != ownership.Workload.GenerationReference { + return ControlledSessionOwnershipV1{}, fmt.Errorf("controlled session workload generation does not match admitted live run %q", ownership.LiveRunID) + } + ownership.BootSession = admitted.BootSession + if err := validateControlledSessionOwnershipV1(ownership); err != nil { + return ControlledSessionOwnershipV1{}, err + } + insert := sort.Search(len(queue.ControlledSessions), func(index int) bool { + return queue.ControlledSessions[index].LiveRunID >= ownership.LiveRunID + }) + if insert < len(queue.ControlledSessions) && queue.ControlledSessions[insert].LiveRunID == ownership.LiveRunID { + merged, err := mergeControlledSessionOwnershipV1(queue.ControlledSessions[insert], ownership) + if err != nil { + return ControlledSessionOwnershipV1{}, fmt.Errorf("live run %q already has different controlled-session ownership: %w", ownership.LiveRunID, err) + } + if merged == queue.ControlledSessions[insert] { + return merged, nil + } + queue.ControlledSessions[insert] = merged + if err := commitLiveRunQueuePathV1(path, queue); err != nil { + return ControlledSessionOwnershipV1{}, err + } + return merged, nil + } + queue.ControlledSessions = append(queue.ControlledSessions, ControlledSessionOwnershipV1{}) + copy(queue.ControlledSessions[insert+1:], queue.ControlledSessions[insert:]) + queue.ControlledSessions[insert] = ownership + if err := commitLiveRunQueuePathV1(path, queue); err != nil { + return ControlledSessionOwnershipV1{}, err + } + return ownership, nil +} + +func mergeControlledSessionOwnershipV1( + existing ControlledSessionOwnershipV1, + requested ControlledSessionOwnershipV1, +) (ControlledSessionOwnershipV1, error) { + existingPlan := existing + requestedPlan := requested + existingPlan.Controller.ID = "" + existingPlan.Workload.ID = "" + requestedPlan.Controller.ID = "" + requestedPlan.Workload.ID = "" + if existingPlan != requestedPlan { + return ControlledSessionOwnershipV1{}, fmt.Errorf("immutable resource plan changed") + } + merged := existing + mergeID := func(current string, next string, role string) (string, error) { + if next == "" { + return current, nil + } + if current != "" && current != next { + return "", fmt.Errorf("%s container ID changed", role) + } + return next, nil + } + var err error + merged.Controller.ID, err = mergeID(existing.Controller.ID, requested.Controller.ID, "controller") + if err != nil { + return ControlledSessionOwnershipV1{}, err + } + merged.Workload.ID, err = mergeID(existing.Workload.ID, requested.Workload.ID, "workload") + if err != nil { + return ControlledSessionOwnershipV1{}, err + } + if err := validateControlledSessionOwnershipV1(merged); err != nil { + return ControlledSessionOwnershipV1{}, err + } + return merged, nil +} + +// CompleteControlledSessionV1 atomically removes a verified-clean session's +// ownership record and admitted run. Failed cleanup must not call this method. +func (lock *OperationLock) CompleteControlledSessionV1(id string) (bool, error) { + if lock == nil { + return false, fmt.Errorf("complete controlled session requires an operation lock") + } + if err := ValidateLiveRunIDV1(id); err != nil { + return false, err + } + lock.mutex.Lock() + defer lock.mutex.Unlock() + path, err := lock.liveRunQueuePathLockedV1() + if err != nil { + return false, err + } + queue, _, err := readLiveRunQueuePathV1(path) + if err != nil { + return false, err + } + updated, runRemoved, err := RemoveLiveRunV1(queue, id) + if err != nil { + return false, err + } + ownershipRemoved := false + for index, ownership := range updated.ControlledSessions { + if ownership.LiveRunID != id { + continue + } + updated.ControlledSessions = append(updated.ControlledSessions[:index], updated.ControlledSessions[index+1:]...) + ownershipRemoved = true + break + } + if !runRemoved && !ownershipRemoved { + return false, nil + } + if err := commitLiveRunQueuePathV1(path, updated); err != nil { + return false, err + } + return true, nil +} + func (lock *OperationLock) RemoveLiveRunV1(id string) (LiveRunQueueV1, bool, error) { if lock == nil { return LiveRunQueueV1{}, false, fmt.Errorf("remove live run requires an operation lock") @@ -518,7 +667,7 @@ func commitLiveRunQueuePathV1(path string, queue LiveRunQueueV1) error { if err != nil { return err } - if len(queue.Runs) == 0 && len(queue.Cleanup) == 0 { + if len(queue.Runs) == 0 && len(queue.ControlledSessions) == 0 && len(queue.Cleanup) == 0 { return removeLiveRunQueuePathV1(path) } if err := writeAtomicStateFile(path, content, 0o600); err != nil { diff --git a/internal/deploy/live_run_queue_file_test.go b/internal/deploy/live_run_queue_file_test.go index 1b4a759..e304796 100644 --- a/internal/deploy/live_run_queue_file_test.go +++ b/internal/deploy/live_run_queue_file_test.go @@ -60,6 +60,155 @@ func TestOperationLockLiveRunQueueFileLifecycle(t *testing.T) { } } +func TestOperationLockRecordsExactControlledSessionOwnership(t *testing.T) { + dir := t.TempDir() + lock, err := AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer lock.Unlock() + const runID = "run-0000000000000001" + const generation = "reploy/env/workload:g-current" + status, err := lock.AdmitLiveRunV1(LiveRunV1{ + ID: runID, Kind: LiveRunKindShellV1, Name: "controlled-session", + GenerationReference: generation, Exclusive: true, + }, false) + if err != nil || status != LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + ownership := controlledSessionOwnershipFixtureV1(dir, runID, generation) + planned := ownership + planned.Controller.ID = "" + planned.Workload.ID = "" + recorded, err := lock.RecordControlledSessionOwnershipV1(planned) + if err != nil { + t.Fatal(err) + } + if recorded.BootSession == "" || planned.BootSession != "" || recorded.Controller.ID != "" || recorded.Workload.ID != "" { + t.Fatalf("boot identity = recorded %q, input %q", recorded.BootSession, ownership.BootSession) + } + if err := validateControlledSessionContainerOwnershipV1(recorded.Controller, "controller"); err == nil || !strings.Contains(err.Error(), "container ID") { + t.Fatalf("complete container validation accepted planned ownership: %v", err) + } + workloadFirst := planned + workloadFirst.Workload.ID = ownership.Workload.ID + if _, err := lock.RecordControlledSessionOwnershipV1(workloadFirst); err == nil || !strings.Contains(err.Error(), "before the controller") { + t.Fatalf("workload-first ownership error = %v", err) + } + controllerPrepared := ownership + controllerPrepared.Workload.ID = "" + recorded, err = lock.RecordControlledSessionOwnershipV1(controllerPrepared) + if err != nil || recorded.Controller.ID != ownership.Controller.ID || recorded.Workload.ID != "" { + t.Fatalf("controller ownership = %#v, error=%v", recorded, err) + } + recorded, err = lock.RecordControlledSessionOwnershipV1(ownership) + if err != nil || recorded.Controller.ID != ownership.Controller.ID || recorded.Workload.ID != ownership.Workload.ID { + t.Fatalf("complete ownership = %#v, error=%v", recorded, err) + } + loaded, found, err := lock.ReadLiveRunQueueV1() + if err != nil || !found || len(loaded.ControlledSessions) != 1 || loaded.ControlledSessions[0] != recorded { + t.Fatalf("controlled-session ownership = %#v, found=%t, error=%v", loaded.ControlledSessions, found, err) + } + conflict := ownership + conflict.Controller.ID = strings.Repeat("c", 64) + if _, err := lock.RecordControlledSessionOwnershipV1(conflict); err == nil || !strings.Contains(err.Error(), "different controlled-session ownership") { + t.Fatalf("conflicting ownership error = %v", err) + } + if completed, err := lock.CompleteControlledSessionV1(runID); err != nil || !completed { + t.Fatalf("completion = %t, %v", completed, err) + } + if _, found, err := lock.ReadLiveRunQueueV1(); err != nil || found { + t.Fatalf("completed queue found=%t, error=%v", found, err) + } +} + +func TestOperationLockControlledSessionOwnershipWriteFailurePreservesQueue(t *testing.T) { + dir := t.TempDir() + lock, err := AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer lock.Unlock() + const runID = "run-0000000000000001" + const generation = "reploy/env/workload:g-current" + if status, err := lock.AdmitLiveRunV1(LiveRunV1{ + ID: runID, Kind: LiveRunKindShellV1, Name: "controlled-session", + GenerationReference: generation, Exclusive: true, + }, false); err != nil || status != LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + planned := controlledSessionOwnershipFixtureV1(dir, runID, generation) + planned.Controller.ID = "" + planned.Workload.ID = "" + if _, err := lock.RecordControlledSessionOwnershipV1(planned); err != nil { + t.Fatal(err) + } + before, _, err := lock.ReadLiveRunQueueV1() + if err != nil { + t.Fatal(err) + } + originalReplace := replaceAtomicStateFile + replaceAtomicStateFile = func(string, string) error { return errors.New("injected ownership replace failure") } + t.Cleanup(func() { replaceAtomicStateFile = originalReplace }) + controllerPrepared := controlledSessionOwnershipFixtureV1(dir, runID, generation) + controllerPrepared.Workload.ID = "" + if _, err := lock.RecordControlledSessionOwnershipV1(controllerPrepared); err == nil || !strings.Contains(err.Error(), "injected ownership replace failure") { + t.Fatalf("ownership write error = %v", err) + } + after, _, err := lock.ReadLiveRunQueueV1() + if err != nil || !reflect.DeepEqual(after, before) { + t.Fatalf("failed ownership write changed queue: %#v, error=%v", after, err) + } +} + +func TestRecoverLiveRunQueuePreservesControlledSessionOwnership(t *testing.T) { + dir := t.TempDir() + lock, err := AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer lock.Unlock() + const runID = "run-0000000000000001" + const generation = "reploy/env/workload:g-current" + if status, err := lock.AdmitLiveRunV1(LiveRunV1{ + ID: runID, Kind: LiveRunKindShellV1, Name: "controlled-session", + GenerationReference: generation, Exclusive: true, + }, false); err != nil || status != LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + recorded, err := lock.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFixtureV1(dir, runID, generation)) + if err != nil { + t.Fatal(err) + } + recovery, err := lock.RecoverLiveRunQueueV1() + if err != nil { + t.Fatal(err) + } + if len(recovery.Removed) != 1 || recovery.Removed[0].Run.ID != runID { + t.Fatalf("recovery = %#v", recovery) + } + queue, found, err := lock.ReadLiveRunQueueV1() + if err != nil || !found || len(queue.Runs) != 0 || len(queue.ControlledSessions) != 1 || queue.ControlledSessions[0] != recorded { + t.Fatalf("ownership after run recovery = %#v, found=%t, error=%v", queue, found, err) + } +} + +func controlledSessionOwnershipFixtureV1(dir string, runID string, generation string) ControlledSessionOwnershipV1 { + container := func(role string, id string, environment string, generation string, build string) ControlledSessionContainerOwnershipV1 { + return ControlledSessionContainerOwnershipV1{ + Role: role, ID: id, Name: "reploy-" + role + "-" + runID, + DeploymentID: environment, GenerationReference: generation, + BuildIdentity: "sha256:" + strings.Repeat(build, 64), + } + } + return ControlledSessionOwnershipV1{ + LiveRunID: runID, SessionHandle: "session-" + strings.Repeat("a", 64), + ChannelDirectory: filepath.Join(dir, ".reploy", "private", "sessions", runID), + Controller: container("controller", strings.Repeat("a", 64), "controller", "reploy/env/controller:g-current", "1"), + Workload: container("workload", strings.Repeat("b", 64), "workload", generation, "2"), + } +} + func TestOperationLockLiveRunQueueReplaceFailurePreservesQueue(t *testing.T) { dir := t.TempDir() lock, err := AcquireOperationLock(t.Context(), dir) diff --git a/internal/dockerdeploy/control_admission_modes.go b/internal/dockerdeploy/control_admission_modes.go index 42473e9..2461eed 100644 --- a/internal/dockerdeploy/control_admission_modes.go +++ b/internal/dockerdeploy/control_admission_modes.go @@ -251,13 +251,13 @@ func stopActiveLiveRunsForControlV1( } stopped := []deploy.LiveRunV1{} for _, run := range active { - if run.Container != "" { + for _, container := range liveRunContainerTargetsV1(queue, run) { err := removeContainer( - TemporaryContainerStopCommand(run.Container), + TemporaryContainerStopCommand(container), RunOptions{Context: ctx, DockerPreflightTimeout: dockerPreflightTimeout}, ) if err != nil && !isMissingContainerCleanupError(err) { - return stopped, fmt.Errorf("stop live run container %q: %w", run.Container, err) + return stopped, fmt.Errorf("stop live run container %q: %w", container, err) } } _, removed, err := operation.RemoveLiveRunV1(run.ID) diff --git a/internal/dockerdeploy/control_admission_modes_test.go b/internal/dockerdeploy/control_admission_modes_test.go index f7daa89..bdc54ed 100644 --- a/internal/dockerdeploy/control_admission_modes_test.go +++ b/internal/dockerdeploy/control_admission_modes_test.go @@ -147,6 +147,123 @@ func TestAdmitControlOperationV1ForceStopsActiveContainersBeforeMarker(t *testin } } +func TestAdmitControlOperationV1ForceStopsControlledSessionContainersAndRetainsOwnership(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + dir := plan.Workload.DeploymentDirectory + operation, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + run := liveRunAdmissionFixtureV1(plan.LiveRunID, false) + run.Kind = deploy.LiveRunKindShellV1 + run.GenerationReference = plan.Workload.GenerationReference + holdLiveRunLeaseV1(t, operation, run.ID) + if _, err := operation.AdmitLiveRunV1(run, false); err != nil { + t.Fatal(err) + } + ownership, err := operation.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFromPlanV1( + plan, dockerControllerTestContainerIDV1, dockerWorkloadTestContainerIDV1, + )) + if err != nil { + t.Fatal(err) + } + + calls := []CommandSpec{} + result, err := admitControlOperationV1(t.Context(), dir, operation, ControlAdmissionInputV1{ + Operation: deploy.ControlOperationStopV1, GenerationReference: run.GenerationReference, + Mode: ControlAdmissionForceV1, + }, controlOperationAdmissionBackendV1{ + newID: func() (string, error) { return "control-0000000000000001", nil }, + pause: func(context.Context, time.Duration) error { return nil }, + await: AwaitControlAdmissionWithNoticeV1, + removeContainer: func(spec CommandSpec, _ RunOptions) error { + calls = append(calls, spec) + return nil + }, + }) + if err != nil || len(result.StoppedRuns) != 1 || result.StoppedRuns[0].ID != run.ID { + t.Fatalf("controlled-session force result = %#v, %v", result, err) + } + wantCalls := []CommandSpec{ + TemporaryContainerStopCommand(dockerWorkloadTestContainerIDV1), + TemporaryContainerStopCommand(dockerControllerTestContainerIDV1), + } + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("controlled-session force calls = %#v", calls) + } + queue, _, err := result.Operation.ReadLiveRunQueueV1() + if err != nil || len(queue.ControlledSessions) != 1 || queue.ControlledSessions[0] != ownership { + t.Fatalf("retained controlled-session ownership = %#v, error=%v", queue.ControlledSessions, err) + } + if len(queue.Runs) != 1 || queue.Runs[0].Kind != deploy.LiveRunKindControlV1 { + t.Fatalf("controlled-session force queue = %#v", queue) + } + if err := CompleteControlAdmissionV1(result.Operation, result.Marker.ID, result.Lease); err != nil { + t.Fatal(err) + } +} + +func TestAdmitControlOperationV1ForcePreservesControlledSessionOnPartialStopFailure(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + dir := plan.Workload.DeploymentDirectory + operation, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + run := liveRunAdmissionFixtureV1(plan.LiveRunID, false) + run.Kind = deploy.LiveRunKindShellV1 + run.GenerationReference = plan.Workload.GenerationReference + holdLiveRunLeaseV1(t, operation, run.ID) + if _, err := operation.AdmitLiveRunV1(run, false); err != nil { + t.Fatal(err) + } + ownership, err := operation.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFromPlanV1( + plan, dockerControllerTestContainerIDV1, dockerWorkloadTestContainerIDV1, + )) + if err != nil { + t.Fatal(err) + } + + want := errors.New("controller stop failed") + calls := []CommandSpec{} + result, err := admitControlOperationV1(t.Context(), dir, operation, ControlAdmissionInputV1{ + Operation: deploy.ControlOperationStopV1, GenerationReference: run.GenerationReference, + Mode: ControlAdmissionForceV1, + }, controlOperationAdmissionBackendV1{ + newID: func() (string, error) { return "control-0000000000000001", nil }, + pause: func(context.Context, time.Duration) error { return nil }, + await: AwaitControlAdmissionWithNoticeV1, + removeContainer: func(spec CommandSpec, _ RunOptions) error { + calls = append(calls, spec) + if len(calls) == 2 { + return want + } + return nil + }, + }) + if !errors.Is(err, want) || len(result.StoppedRuns) != 0 { + t.Fatalf("partial controlled-session stop result = %#v, %v", result, err) + } + wantCalls := []CommandSpec{ + TemporaryContainerStopCommand(dockerWorkloadTestContainerIDV1), + TemporaryContainerStopCommand(dockerControllerTestContainerIDV1), + } + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("partial controlled-session stop calls = %#v", calls) + } + check, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + queue, _, err := check.ReadLiveRunQueueV1() + if err != nil || len(queue.Runs) != 1 || queue.Runs[0].ID != run.ID || + len(queue.ControlledSessions) != 1 || queue.ControlledSessions[0] != ownership || + len(deploy.ControlMarkersV1(queue)) != 0 { + t.Fatalf("queue after partial controlled-session stop = %#v, error=%v", queue, err) + } +} + func TestAdmitControlOperationV1ForceFailurePreservesFailedAndLaterActiveRuns(t *testing.T) { dir := t.TempDir() operation, err := deploy.AcquireOperationLock(t.Context(), dir) diff --git a/internal/dockerdeploy/controlled_session_controller.go b/internal/dockerdeploy/controlled_session_controller.go index 1d6fb17..7f30033 100644 --- a/internal/dockerdeploy/controlled_session_controller.go +++ b/internal/dockerdeploy/controlled_session_controller.go @@ -40,6 +40,10 @@ type DockerControllerV1 struct { waitResult dockerControllerWaitResultV1 } +func (controller *DockerControllerV1) ContainerID() string { + return controller.containerID +} + // PrepareDockerControllerV1 verifies that the private channel is ready and // creates the exact controller container without starting it. func PrepareDockerControllerV1( diff --git a/internal/dockerdeploy/controlled_session_supervisor.go b/internal/dockerdeploy/controlled_session_supervisor.go index 1daa4c7..0a0263e 100644 --- a/internal/dockerdeploy/controlled_session_supervisor.go +++ b/internal/dockerdeploy/controlled_session_supervisor.go @@ -5,10 +5,13 @@ import ( "errors" "fmt" "io" + "os" + "path/filepath" "sync" "time" "github.com/omry/reploy/internal/controlledsession" + "github.com/omry/reploy/internal/deploy" ) const controlledSessionOutputFinalizationTimeoutV1 = time.Duration(controlledsession.DefaultOutputFinalizationTimeoutMillisecondsV1) * time.Millisecond @@ -34,6 +37,7 @@ type ControlledSessionRunResultV1 struct { } type controlledSessionControllerRuntimeV1 interface { + ContainerID() string Start(context.Context) error Wait(context.Context) (controlledsession.ProcessStatusV1, error) RequestGracefulStop(context.Context) error @@ -43,6 +47,7 @@ type controlledSessionControllerRuntimeV1 interface { type controlledSessionWorkloadRuntimeV1 interface { controlledsession.WorkloadPTYControlV1 + ContainerID() string Output() (io.ReadCloser, error) Start(context.Context) error Started() bool @@ -71,10 +76,13 @@ func (runtime *privateControlledSessionChannelRuntimeV1) Close() error { } type controlledSessionSupervisorBackendV1 struct { - prepareChannel func(ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) - prepareController func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionControllerRuntimeV1, error) - prepareWorkload func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) - now func() time.Time + prepareChannel func(ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) + prepareController func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionControllerRuntimeV1, error) + prepareWorkload func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) + recordPlannedOwnership func() error + recordControllerOwnership func(string) error + recordOwnership func(string, string) error + now func() time.Time } type controlledSessionProcessResultV1 struct { @@ -120,19 +128,57 @@ type controlledSessionSupervisorV1 struct { diagnosticErr error } -// RunControlledSessionV1 owns one attached controller/workload operation from -// inert resource creation through terminal acknowledgement and ordinary -// delivery-tail cleanup. Crash reconciliation, watchdog ownership, networking, -// and public command exposure are deliberately outside this lifecycle core. +// RunControlledSessionV1 takes ownership of the admitted workload operation +// lock. The caller must retain the live-run queue-entry lease until this call +// returns. The supervisor durably records both exact inert containers and the +// private channel before releasing the lock and starting either process. func RunControlledSessionV1( ctx context.Context, + operation *deploy.OperationLock, plan ControlledSessionExecutionPlanV1, options ControlledSessionRunOptionsV1, ) (ControlledSessionRunResultV1, error) { - return runControlledSessionV1(ctx, plan, options, controlledSessionSupervisorBackendV1{ + if operation == nil { + return ControlledSessionRunResultV1{}, fmt.Errorf("run controlled session requires an admitted operation lock") + } + if err := operation.RequireHeld(); err != nil { + return ControlledSessionRunResultV1{}, err + } + absoluteDir, err := filepath.Abs(plan.Workload.DeploymentDirectory) + if err != nil { + return ControlledSessionRunResultV1{}, releaseControlledSessionOperationV1(operation, fmt.Errorf("resolve controlled-session workload deployment directory: %w", err)) + } + if filepath.Dir(filepath.Dir(operation.Path())) != absoluteDir { + return ControlledSessionRunResultV1{}, removeUnstartedControlledSessionV1(operation, plan.LiveRunID, fmt.Errorf("controlled-session operation lock does not belong to workload deployment %q", absoluteDir)) + } + if ctx == nil || ctx.Done() == nil { + return ControlledSessionRunResultV1{}, removeUnstartedControlledSessionV1(operation, plan.LiveRunID, fmt.Errorf("run controlled session: cancelable host context is required")) + } + if err := ValidateControlledSessionExecutionPlanV1(plan); err != nil { + return ControlledSessionRunResultV1{}, removeUnstartedControlledSessionV1(operation, plan.LiveRunID, fmt.Errorf("run controlled session plan: %w", err)) + } + if err := validateControlledSessionRunOptionsV1(options); err != nil { + return ControlledSessionRunResultV1{}, removeUnstartedControlledSessionV1(operation, plan.LiveRunID, err) + } + if err := operation.RequireQueueEntryLeaseHeldV1(plan.LiveRunID); err != nil { + return ControlledSessionRunResultV1{}, removeUnstartedControlledSessionV1(operation, plan.LiveRunID, fmt.Errorf("controlled-session admission ownership: %w", err)) + } + released := false + ownershipRecorded := false + partialPreparationCleanupVerified := false + persistOwnership := func(controllerID string, workloadID string) error { + ownership := controlledSessionOwnershipFromPlanV1(plan, controllerID, workloadID) + if _, err := operation.RecordControlledSessionOwnershipV1(ownership); err != nil { + return fmt.Errorf("persist controlled-session ownership: %w", err) + } + ownershipRecorded = true + return nil + } + result, runErr := runControlledSessionV1(ctx, plan, options, controlledSessionSupervisorBackendV1{ prepareChannel: func(plan ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) { channel, err := PrepareControlledSessionChannelV1(plan) if err != nil { + partialPreparationCleanupVerified = controlledSessionChannelAbsentV1(plan.Channel.HostDirectory) return nil, err } return &privateControlledSessionChannelRuntimeV1{channel: channel}, nil @@ -143,8 +189,96 @@ func RunControlledSessionV1( prepareWorkload: func(ctx context.Context, plan ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) { return PrepareDockerWorkloadPTYV1(ctx, plan) }, + recordPlannedOwnership: func() error { + return persistOwnership("", "") + }, + recordControllerOwnership: func(controllerID string) error { + return persistOwnership(controllerID, "") + }, + recordOwnership: func(controllerID string, workloadID string) error { + if err := persistOwnership(controllerID, workloadID); err != nil { + return err + } + if err := operation.Unlock(); err != nil { + return fmt.Errorf("release operation lock before controlled-session startup: %w", err) + } + released = true + return nil + }, now: time.Now, }) + cleaned := result.SessionResult.CleanupStatus.Kind == controlledsession.CleanupStatusSucceededV1 && + result.DeliveryTailCleanupStatus.Kind == controlledsession.CleanupStatusSucceededV1 + cleaned = controlledSessionPreparationCanCompleteV1(cleaned, ownershipRecorded, released, partialPreparationCleanupVerified) + completionErr := finishControlledSessionOwnershipV1(context.WithoutCancel(ctx), absoluteDir, operation, released, plan.LiveRunID, cleaned) + return result, errors.Join(runErr, completionErr) +} + +func controlledSessionChannelAbsentV1(path string) bool { + _, err := os.Lstat(path) + return errors.Is(err, os.ErrNotExist) +} + +func controlledSessionPreparationCanCompleteV1(cleaned bool, ownershipRecorded bool, released bool, channelCleanupVerified bool) bool { + return cleaned && (!ownershipRecorded || released || channelCleanupVerified) +} + +func controlledSessionOwnershipFromPlanV1(plan ControlledSessionExecutionPlanV1, controllerID string, workloadID string) deploy.ControlledSessionOwnershipV1 { + container := func(plan ControlledSessionContainerPlanV1, id string) deploy.ControlledSessionContainerOwnershipV1 { + return deploy.ControlledSessionContainerOwnershipV1{ + Role: string(plan.Role), ID: id, Name: plan.Container, DeploymentID: plan.DeploymentID, + GenerationReference: plan.GenerationReference, BuildIdentity: string(plan.BuildIdentity), + } + } + return deploy.ControlledSessionOwnershipV1{ + LiveRunID: plan.LiveRunID, SessionHandle: plan.Authorization.Handle, + ChannelDirectory: plan.Channel.HostDirectory, + Controller: container(plan.Controller, controllerID), Workload: container(plan.Workload, workloadID), + } +} + +func finishControlledSessionOwnershipV1( + ctx context.Context, + deploymentDir string, + operation *deploy.OperationLock, + released bool, + runID string, + cleaned bool, +) error { + if released { + var err error + operation, err = deploy.AcquireOperationLock(ctx, deploymentDir) + if err != nil { + return fmt.Errorf("reacquire operation lock after controlled session: %w", err) + } + } + var completionErr error + if cleaned { + _, completionErr = operation.CompleteControlledSessionV1(runID) + if completionErr != nil { + completionErr = fmt.Errorf("remove verified-clean controlled-session ownership: %w", completionErr) + } + } + unlockErr := operation.Unlock() + if unlockErr != nil { + unlockErr = fmt.Errorf("release controlled-session operation lock: %w", unlockErr) + } + return errors.Join(completionErr, unlockErr) +} + +func removeUnstartedControlledSessionV1(operation *deploy.OperationLock, runID string, cause error) error { + var removeErr error + if deploy.ValidateLiveRunIDV1(runID) == nil { + _, _, removeErr = operation.RemoveLiveRunV1(runID) + } + return releaseControlledSessionOperationV1(operation, errors.Join(cause, removeErr)) +} + +func releaseControlledSessionOperationV1(operation *deploy.OperationLock, cause error) error { + if err := operation.Unlock(); err != nil { + return errors.Join(cause, fmt.Errorf("release controlled-session operation lock: %w", err)) + } + return cause } func runControlledSessionV1( @@ -165,6 +299,12 @@ func runControlledSessionV1( if backend.prepareChannel == nil || backend.prepareController == nil || backend.prepareWorkload == nil || backend.now == nil { return ControlledSessionRunResultV1{}, fmt.Errorf("run controlled session: supervisor backend is incomplete") } + ownershipCallbacksEnabled := backend.recordPlannedOwnership != nil || + backend.recordControllerOwnership != nil || backend.recordOwnership != nil + if ownershipCallbacksEnabled && (backend.recordPlannedOwnership == nil || + backend.recordControllerOwnership == nil || backend.recordOwnership == nil) { + return ControlledSessionRunResultV1{}, fmt.Errorf("run controlled session: ownership backend is incomplete") + } machine, err := controlledsession.NewMachineV1(plan.Authorization) if err != nil { return ControlledSessionRunResultV1{}, fmt.Errorf("run controlled session lifecycle: %w", err) @@ -242,6 +382,11 @@ func (supervisor *controlledSessionSupervisorV1) run(ctx context.Context) (Contr } func (supervisor *controlledSessionSupervisorV1) prepare(ctx context.Context) error { + if supervisor.backend.recordPlannedOwnership != nil { + if err := supervisor.backend.recordPlannedOwnership(); err != nil { + return err + } + } channel, err := supervisor.backend.prepareChannel(supervisor.plan) if err != nil { return fmt.Errorf("prepare controlled-session channel: %w", err) @@ -253,6 +398,11 @@ func (supervisor *controlledSessionSupervisorV1) prepare(ctx context.Context) er return fmt.Errorf("prepare controlled-session controller: %w", err) } supervisor.controller = controller + if supervisor.backend.recordControllerOwnership != nil { + if err := supervisor.backend.recordControllerOwnership(controller.ContainerID()); err != nil { + return err + } + } workload, err := supervisor.backend.prepareWorkload(ctx, supervisor.plan.Workload) if err != nil { return fmt.Errorf("prepare controlled-session workload: %w", err) @@ -262,6 +412,11 @@ func (supervisor *controlledSessionSupervisorV1) prepare(ctx context.Context) er if err != nil { return fmt.Errorf("claim controlled-session workload output: %w", err) } + if supervisor.backend.recordOwnership != nil { + if err := supervisor.backend.recordOwnership(controller.ContainerID(), workload.ContainerID()); err != nil { + return err + } + } if err := controller.Start(ctx); err != nil { return fmt.Errorf("start controlled-session controller: %w", err) } diff --git a/internal/dockerdeploy/controlled_session_supervisor_integration_test.go b/internal/dockerdeploy/controlled_session_supervisor_integration_test.go index 7240a39..7dc62e1 100644 --- a/internal/dockerdeploy/controlled_session_supervisor_integration_test.go +++ b/internal/dockerdeploy/controlled_session_supervisor_integration_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/omry/reploy/internal/controlledsession" + "github.com/omry/reploy/internal/deploy" ) func TestControlledSessionSupervisorDockerIntegration(t *testing.T) { @@ -23,8 +24,32 @@ func TestControlledSessionSupervisorDockerIntegration(t *testing.T) { defer cancel() image := buildControlledSessionControllerIntegrationImageV1(t, ctx) plan := controlledSessionControllerIntegrationPlanV1(t, image, []string{"/session-channel-helper", "supervise"}) + operation, err := deploy.AcquireOperationLock(ctx, plan.Workload.DeploymentDirectory) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = operation.Unlock() }) + lease, err := operation.AcquireLiveRunLeaseV1(plan.LiveRunID) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := lease.Release(); err != nil { + t.Errorf("release controlled-session live-run lease: %v", err) + } + }) + status, err := operation.AdmitLiveRunV1(deploy.LiveRunV1{ + ID: plan.LiveRunID, Kind: deploy.LiveRunKindShellV1, Name: plan.Workload.DeploymentID, + GenerationReference: plan.Workload.GenerationReference, Exclusive: true, + }, false) + if err != nil { + t.Fatal(err) + } + if status != deploy.LiveRunStatusActiveV1 { + t.Fatalf("controlled-session live run status = %q", status) + } - result, err := RunControlledSessionV1(ctx, plan, ControlledSessionRunOptionsV1{ + result, err := RunControlledSessionV1(ctx, operation, plan, ControlledSessionRunOptionsV1{ StartupTimeout: 30 * time.Second, TerminationGrace: 5 * time.Second, ControllerFinalizationTimeout: 15 * time.Second, ResultAcknowledgementTimeout: 5 * time.Second, CleanupTimeout: 15 * time.Second, @@ -53,4 +78,12 @@ func TestControlledSessionSupervisorDockerIntegration(t *testing.T) { if _, statErr := os.Stat(plan.Channel.HostDirectory); !os.IsNotExist(statErr) { t.Fatalf("private channel directory survived cleanup: %v", statErr) } + check, lockErr := deploy.AcquireOperationLock(ctx, plan.Workload.DeploymentDirectory) + if lockErr != nil { + t.Fatal(lockErr) + } + defer check.Unlock() + if queue, found, readErr := check.ReadLiveRunQueueV1(); readErr != nil || found { + t.Fatalf("verified-clean session retained ownership: %#v, found=%t, error=%v", queue, found, readErr) + } } diff --git a/internal/dockerdeploy/controlled_session_supervisor_test.go b/internal/dockerdeploy/controlled_session_supervisor_test.go index 67443e4..e95f13e 100644 --- a/internal/dockerdeploy/controlled_session_supervisor_test.go +++ b/internal/dockerdeploy/controlled_session_supervisor_test.go @@ -4,11 +4,14 @@ import ( "context" "errors" "io" + "reflect" + "strings" "sync" "testing" "time" "github.com/omry/reploy/internal/controlledsession" + "github.com/omry/reploy/internal/deploy" ) func TestRunControlledSessionV1OwnsNormalLifecycle(t *testing.T) { @@ -91,6 +94,267 @@ func TestRunControlledSessionV1OwnsNormalLifecycle(t *testing.T) { } } +func TestRunControlledSessionV1PersistsExactOwnershipBeforeStarting(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + controller := newFakeControlledSessionProcessV1() + workload := newFakeControlledSessionWorkloadV1(nil, 0) + channel := &fakeControlledSessionChannelV1{} + persistErr := errors.New("injected durable ownership failure") + calls := []string{} + + result, err := runControlledSessionV1(t.Context(), plan, testControlledSessionRunOptionsV1(), controlledSessionSupervisorBackendV1{ + prepareChannel: func(ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) { + calls = append(calls, "channel") + return channel, nil + }, + prepareController: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionControllerRuntimeV1, error) { + calls = append(calls, "prepare-controller") + return controller, nil + }, + prepareWorkload: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) { + calls = append(calls, "prepare-workload") + return workload, nil + }, + recordPlannedOwnership: func() error { + calls = append(calls, "planned") + if controller.started || workload.started { + t.Fatal("controlled-session process started before planned ownership") + } + return nil + }, + recordControllerOwnership: func(controllerID string) error { + calls = append(calls, "controller") + if controllerID != dockerControllerTestContainerIDV1 { + t.Fatalf("controller ID = %q", controllerID) + } + return nil + }, + recordOwnership: func(controllerID string, workloadID string) error { + calls = append(calls, "complete") + if controller.started || workload.started { + t.Fatal("controlled-session process started before durable ownership") + } + if controllerID != dockerControllerTestContainerIDV1 || workloadID != dockerWorkloadTestContainerIDV1 { + t.Fatalf("container IDs = %q / %q", controllerID, workloadID) + } + return persistErr + }, + now: time.Now, + }) + if !reflect.DeepEqual(calls, []string{"planned", "channel", "prepare-controller", "controller", "prepare-workload", "complete"}) || !errors.Is(err, persistErr) { + t.Fatalf("ownership persistence calls=%v, error=%v", calls, err) + } + if controller.started || workload.started { + t.Fatalf("started after persistence failure = controller %t workload %t", controller.started, workload.started) + } + if !controller.cleaned || !workload.cleaned || !channel.closed { + t.Fatalf("inert cleanup = controller %t workload %t channel %t", controller.cleaned, workload.cleaned, channel.closed) + } + if result.SessionResult.CleanupStatus.Kind != controlledsession.CleanupStatusSucceededV1 || + result.DeliveryTailCleanupStatus.Kind != controlledsession.CleanupStatusSucceededV1 { + t.Fatalf("cleanup result = %#v", result) + } +} + +func TestRunControlledSessionV1RecordsControllerOwnershipBeforeWorkloadPreparationFailure(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + controller := newFakeControlledSessionProcessV1() + channel := &fakeControlledSessionChannelV1{} + prepareErr := errors.New("injected workload preparation failure") + calls := []string{} + + result, err := runControlledSessionV1(t.Context(), plan, testControlledSessionRunOptionsV1(), controlledSessionSupervisorBackendV1{ + prepareChannel: func(ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) { + calls = append(calls, "channel") + return channel, nil + }, + prepareController: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionControllerRuntimeV1, error) { + calls = append(calls, "prepare-controller") + return controller, nil + }, + prepareWorkload: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) { + calls = append(calls, "prepare-workload") + return nil, prepareErr + }, + recordPlannedOwnership: func() error { + calls = append(calls, "planned") + return nil + }, + recordControllerOwnership: func(controllerID string) error { + calls = append(calls, "controller") + if controllerID != dockerControllerTestContainerIDV1 { + t.Fatalf("controller ID = %q", controllerID) + } + return nil + }, + recordOwnership: func(string, string) error { + t.Fatal("complete ownership recorded without a workload") + return nil + }, + now: time.Now, + }) + if !errors.Is(err, prepareErr) { + t.Fatalf("workload preparation error = %v", err) + } + if !reflect.DeepEqual(calls, []string{"planned", "channel", "prepare-controller", "controller", "prepare-workload"}) { + t.Fatalf("partial ownership calls = %v", calls) + } + if controller.started { + t.Fatal("controller started after workload preparation failed") + } + if !controller.cleaned || !channel.closed { + t.Fatalf("partial preparation cleanup = controller %t channel %t", controller.cleaned, channel.closed) + } + if result.SessionResult.CleanupStatus.Kind != controlledsession.CleanupStatusSucceededV1 || + result.DeliveryTailCleanupStatus.Kind != controlledsession.CleanupStatusSucceededV1 { + t.Fatalf("cleanup result = %#v", result) + } +} + +func TestRunControlledSessionV1RejectsIncompleteOwnershipBackend(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + called := false + _, err := runControlledSessionV1(t.Context(), plan, testControlledSessionRunOptionsV1(), controlledSessionSupervisorBackendV1{ + prepareChannel: func(ControlledSessionExecutionPlanV1) (controlledSessionChannelRuntimeV1, error) { + called = true + return nil, nil + }, + prepareController: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionControllerRuntimeV1, error) { + called = true + return nil, nil + }, + prepareWorkload: func(context.Context, ControlledSessionContainerPlanV1) (controlledSessionWorkloadRuntimeV1, error) { + called = true + return nil, nil + }, + recordOwnership: func(string, string) error { return nil }, + now: time.Now, + }) + if err == nil || !strings.Contains(err.Error(), "ownership backend is incomplete") { + t.Fatalf("incomplete ownership backend error = %v", err) + } + if called { + t.Fatal("incomplete ownership backend began preparation") + } +} + +func TestControlledSessionChannelAbsentV1(t *testing.T) { + existing := t.TempDir() + if controlledSessionChannelAbsentV1(existing) { + t.Fatal("existing channel directory reported absent") + } + if !controlledSessionChannelAbsentV1(existing + "/missing") { + t.Fatal("missing channel directory not reported absent") + } +} + +func TestControlledSessionPreparationCanCompleteV1(t *testing.T) { + tests := []struct { + name string + cleaned bool + ownershipRecorded bool + released bool + channelCleanupVerified bool + want bool + }{ + {name: "cleanup failed", ownershipRecorded: true, released: true}, + {name: "no ownership recorded", cleaned: true, want: true}, + {name: "full ownership released", cleaned: true, ownershipRecorded: true, released: true, want: true}, + {name: "partial ownership ambiguous", cleaned: true, ownershipRecorded: true}, + {name: "channel absence verified", cleaned: true, ownershipRecorded: true, channelCleanupVerified: true, want: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := controlledSessionPreparationCanCompleteV1(test.cleaned, test.ownershipRecorded, test.released, test.channelCleanupVerified); got != test.want { + t.Fatalf("completion = %t, want %t", got, test.want) + } + }) + } +} + +func TestControlledSessionCleanupFailureRetainsDurableOwnership(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + operation, err := deploy.AcquireOperationLock(t.Context(), plan.Workload.DeploymentDirectory) + if err != nil { + t.Fatal(err) + } + if status, err := operation.AdmitLiveRunV1(deploy.LiveRunV1{ + ID: plan.LiveRunID, Kind: deploy.LiveRunKindShellV1, Name: plan.Workload.DeploymentID, + GenerationReference: plan.Workload.GenerationReference, Exclusive: true, + }, false); err != nil || status != deploy.LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + if _, err := operation.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFromPlanV1( + plan, dockerControllerTestContainerIDV1, dockerWorkloadTestContainerIDV1, + )); err != nil { + t.Fatal(err) + } + if err := finishControlledSessionOwnershipV1(t.Context(), plan.Workload.DeploymentDirectory, operation, false, plan.LiveRunID, false); err != nil { + t.Fatal(err) + } + check, err := deploy.AcquireOperationLock(t.Context(), plan.Workload.DeploymentDirectory) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + queue, found, err := check.ReadLiveRunQueueV1() + if err != nil || !found || len(queue.Runs) != 1 || len(queue.ControlledSessions) != 1 { + t.Fatalf("retained queue = %#v, found=%t, error=%v", queue, found, err) + } +} + +func TestRunControlledSessionV1RemovesAdmissionOnLockDirectoryMismatch(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + workloadDir := plan.Workload.DeploymentDirectory + operation, err := deploy.AcquireOperationLock(t.Context(), workloadDir) + if err != nil { + t.Fatal(err) + } + if status, err := operation.AdmitLiveRunV1(deploy.LiveRunV1{ + ID: plan.LiveRunID, Kind: deploy.LiveRunKindShellV1, Name: plan.Workload.DeploymentID, + GenerationReference: plan.Workload.GenerationReference, Exclusive: true, + }, false); err != nil || status != deploy.LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + plan.Workload.DeploymentDirectory = t.TempDir() + if _, err := RunControlledSessionV1(t.Context(), operation, plan, testControlledSessionRunOptionsV1()); err == nil || !strings.Contains(err.Error(), "does not belong") { + t.Fatalf("lock-directory mismatch error = %v", err) + } + check, err := deploy.AcquireOperationLock(t.Context(), workloadDir) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + if queue, found, err := check.ReadLiveRunQueueV1(); err != nil || found { + t.Fatalf("mismatched operation retained admission: %#v, found=%t, error=%v", queue, found, err) + } +} + +func TestRunControlledSessionV1RequiresQueueEntryLease(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + operation, err := deploy.AcquireOperationLock(t.Context(), plan.Workload.DeploymentDirectory) + if err != nil { + t.Fatal(err) + } + if status, err := operation.AdmitLiveRunV1(deploy.LiveRunV1{ + ID: plan.LiveRunID, Kind: deploy.LiveRunKindShellV1, Name: plan.Workload.DeploymentID, + GenerationReference: plan.Workload.GenerationReference, Exclusive: true, + }, false); err != nil || status != deploy.LiveRunStatusActiveV1 { + t.Fatalf("admission = %q, %v", status, err) + } + if _, err := RunControlledSessionV1(t.Context(), operation, plan, testControlledSessionRunOptionsV1()); err == nil || !strings.Contains(err.Error(), "queue-entry lease") { + t.Fatalf("missing queue-entry lease error = %v", err) + } + check, err := deploy.AcquireOperationLock(t.Context(), plan.Workload.DeploymentDirectory) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + if queue, found, err := check.ReadLiveRunQueueV1(); err != nil || found { + t.Fatalf("missing-lease operation retained admission: %#v, found=%t, error=%v", queue, found, err) + } +} + func TestRunControlledSessionV1HoldsCompleteUntilOutputFinalizationPublication(t *testing.T) { plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) requests := make(chan controlledsession.RequestV1, 8) @@ -786,6 +1050,10 @@ type fakeControlledSessionProcessV1 struct { cleaned bool } +func (process *fakeControlledSessionProcessV1) ContainerID() string { + return dockerControllerTestContainerIDV1 +} + func newFakeControlledSessionProcessV1() *fakeControlledSessionProcessV1 { return &fakeControlledSessionProcessV1{exit: make(chan controlledSessionProcessResultV1, 1)} } @@ -843,6 +1111,10 @@ type fakeControlledSessionWorkloadV1 struct { rows uint32 } +func (workload *fakeControlledSessionWorkloadV1) ContainerID() string { + return dockerWorkloadTestContainerIDV1 +} + func newFakeControlledSessionWorkloadV1(output []byte, exitCode int) *fakeControlledSessionWorkloadV1 { reader, writer := io.Pipe() return &fakeControlledSessionWorkloadV1{ diff --git a/internal/dockerdeploy/controlled_session_workload_pty.go b/internal/dockerdeploy/controlled_session_workload_pty.go index f0fea71..37acc93 100644 --- a/internal/dockerdeploy/controlled_session_workload_pty.go +++ b/internal/dockerdeploy/controlled_session_workload_pty.go @@ -54,6 +54,10 @@ type DockerWorkloadPTYV1 struct { closeErr error } +func (workload *DockerWorkloadPTYV1) ContainerID() string { + return workload.containerID +} + // PrepareDockerWorkloadPTYV1 creates the exact workload container without // starting it and establishes the Docker PTY attachment before returning. func PrepareDockerWorkloadPTYV1( diff --git a/internal/dockerdeploy/live_runs.go b/internal/dockerdeploy/live_runs.go index 9667bbc..870bb1b 100644 --- a/internal/dockerdeploy/live_runs.go +++ b/internal/dockerdeploy/live_runs.go @@ -130,13 +130,15 @@ func stopLiveRunV1( } result.Found = true result.Run = run - if run.Status == deploy.LiveRunStatusActiveV1 && run.Container != "" { - removeErr := backend.removeContainer( - TemporaryContainerCleanupCommand(run.Container), - RunOptions{Context: ctx, DockerPreflightTimeout: dockerPreflightTimeout}, - ) - if removeErr != nil && !isMissingContainerCleanupError(removeErr) { - return result, fmt.Errorf("stop live run container %q: %w", run.Container, removeErr) + if run.Status == deploy.LiveRunStatusActiveV1 { + for _, container := range liveRunContainerTargetsV1(queue, run) { + removeErr := backend.removeContainer( + TemporaryContainerCleanupCommand(container), + RunOptions{Context: ctx, DockerPreflightTimeout: dockerPreflightTimeout}, + ) + if removeErr != nil && !isMissingContainerCleanupError(removeErr) { + return result, fmt.Errorf("stop live run container %q: %w", container, removeErr) + } } } _, removed, err := operation.RemoveLiveRunV1(id) @@ -151,3 +153,21 @@ func stopLiveRunV1( } return result, nil } + +// liveRunContainerTargetsV1 returns every exact container owned by a live run. +// Workload-first ordering leaves the controller available to observe workload +// termination for as long as possible. Controlled-session ownership remains +// durable until its supervisor or recovery verifies complete cleanup. +func liveRunContainerTargetsV1(queue deploy.LiveRunQueueV1, run deploy.LiveRunV1) []string { + targets := make([]string, 0, 2) + if run.Container != "" { + targets = append(targets, run.Container) + } + for _, ownership := range queue.ControlledSessions { + if ownership.LiveRunID == run.ID { + targets = append(targets, ownership.Workload.ID, ownership.Controller.ID) + break + } + } + return targets +} diff --git a/internal/dockerdeploy/live_runs_test.go b/internal/dockerdeploy/live_runs_test.go index 40fe842..0071062 100644 --- a/internal/dockerdeploy/live_runs_test.go +++ b/internal/dockerdeploy/live_runs_test.go @@ -133,6 +133,120 @@ func TestStopLiveRunV1RemovesActiveContainerBeforePromotingWaiter(t *testing.T) } } +func TestStopLiveRunV1RemovesControlledSessionContainersAndRetainsOwnership(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + dir := plan.Workload.DeploymentDirectory + operation, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + run := liveRunAdmissionFixtureV1(plan.LiveRunID, false) + run.Kind = deploy.LiveRunKindShellV1 + run.GenerationReference = plan.Workload.GenerationReference + holdLiveRunLeaseV1(t, operation, run.ID) + if _, err := operation.AdmitLiveRunV1(run, false); err != nil { + t.Fatal(err) + } + ownership, err := operation.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFromPlanV1( + plan, dockerControllerTestContainerIDV1, dockerWorkloadTestContainerIDV1, + )) + if err != nil { + t.Fatal(err) + } + if err := operation.Unlock(); err != nil { + t.Fatal(err) + } + + calls := []CommandSpec{} + result, err := stopLiveRunV1(t.Context(), dir, run.ID, 7*time.Second, liveRunsBackendV1{ + acquire: deploy.AcquireOperationLock, + removeContainer: func(spec CommandSpec, options RunOptions) error { + if options.DockerPreflightTimeout != 7*time.Second { + t.Fatalf("Docker timeout = %s", options.DockerPreflightTimeout) + } + calls = append(calls, spec) + return nil + }, + }) + if err != nil || !result.Found || result.Run.ID != run.ID { + t.Fatalf("controlled-session stop = %#v, %v", result, err) + } + wantCalls := []CommandSpec{ + TemporaryContainerCleanupCommand(dockerWorkloadTestContainerIDV1), + TemporaryContainerCleanupCommand(dockerControllerTestContainerIDV1), + } + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("controlled-session cleanup calls = %#v", calls) + } + check, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + queue, found, err := check.ReadLiveRunQueueV1() + if err != nil || !found || len(queue.Runs) != 0 || len(queue.ControlledSessions) != 1 || queue.ControlledSessions[0] != ownership { + t.Fatalf("retained controlled-session ownership = %#v, found=%t, error=%v", queue, found, err) + } +} + +func TestStopLiveRunV1PreservesControlledSessionOnPartialCleanupFailure(t *testing.T) { + plan := controlledSessionControllerIntegrationPlanV1(t, "test-image", []string{"/controller"}) + dir := plan.Workload.DeploymentDirectory + operation, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + run := liveRunAdmissionFixtureV1(plan.LiveRunID, false) + run.Kind = deploy.LiveRunKindShellV1 + run.GenerationReference = plan.Workload.GenerationReference + holdLiveRunLeaseV1(t, operation, run.ID) + if _, err := operation.AdmitLiveRunV1(run, false); err != nil { + t.Fatal(err) + } + ownership, err := operation.RecordControlledSessionOwnershipV1(controlledSessionOwnershipFromPlanV1( + plan, dockerControllerTestContainerIDV1, dockerWorkloadTestContainerIDV1, + )) + if err != nil { + t.Fatal(err) + } + if err := operation.Unlock(); err != nil { + t.Fatal(err) + } + + want := errors.New("controller cleanup failed") + calls := []CommandSpec{} + result, err := stopLiveRunV1(t.Context(), dir, run.ID, 7*time.Second, liveRunsBackendV1{ + acquire: deploy.AcquireOperationLock, + removeContainer: func(spec CommandSpec, _ RunOptions) error { + calls = append(calls, spec) + if len(calls) == 2 { + return want + } + return nil + }, + }) + if !errors.Is(err, want) || !result.Found || result.Run.ID != run.ID { + t.Fatalf("partial controlled-session cleanup = %#v, %v", result, err) + } + wantCalls := []CommandSpec{ + TemporaryContainerCleanupCommand(dockerWorkloadTestContainerIDV1), + TemporaryContainerCleanupCommand(dockerControllerTestContainerIDV1), + } + if !reflect.DeepEqual(calls, wantCalls) { + t.Fatalf("partial controlled-session cleanup calls = %#v", calls) + } + check, err := deploy.AcquireOperationLock(t.Context(), dir) + if err != nil { + t.Fatal(err) + } + defer check.Unlock() + queue, _, err := check.ReadLiveRunQueueV1() + if err != nil || len(queue.Runs) != 1 || queue.Runs[0].ID != run.ID || + len(queue.ControlledSessions) != 1 || queue.ControlledSessions[0] != ownership { + t.Fatalf("queue after partial controlled-session cleanup = %#v, error=%v", queue, err) + } +} + func TestStopLiveRunV1ReportsReadyReservationAsWaitingWithoutDocker(t *testing.T) { dir := t.TempDir() operation, err := deploy.AcquireOperationLock(t.Context(), dir)