demo working

cmd/containerd-shim-runhcs-v1/exec.go

@@ -83,6 +83,7 @@ type shimExec interface {
 	//
 	// `ForceExit` is safe to call in any `State()`.
 	ForceExit(ctx context.Context, status int)
+	Save(ctx context.Context, path string) error
 }
 
 func newExecInvalidStateError(tid, eid string, state shimExecState, op string) error {

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -4,6 +4,8 @@ package main
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -42,7 +44,8 @@ func newHcsExec(
 	id, bundle string,
 	isWCOW bool,
 	spec *specs.Process,
-	io cmd.UpstreamIO) shimExec {
+	io cmd.UpstreamIO,
+) shimExec {
 	log.G(ctx).WithFields(logrus.Fields{
 		"tid":    tid,
 		"eid":    id, // Init exec ID is always same as Task ID
@@ -69,6 +72,63 @@ func newHcsExec(
 	return he
 }
 
+func restoreHcsExec(
+	ctx context.Context,
+	events publisher,
+	tid string,
+	host *uvm.UtilityVM,
+	c cow.Container,
+	id, bundle string,
+	isWCOW bool,
+	spec *specs.Process,
+	io cmd.UpstreamIO,
+	path string,
+) (shimExec, error) {
+	log.G(ctx).WithFields(logrus.Fields{
+		"tid":    tid,
+		"eid":    id, // Init exec ID is always same as Task ID
+		"bundle": bundle,
+		"wcow":   isWCOW,
+	}).Debug("restoreHcsExec")
+
+	he := &hcsExec{
+		events:      events,
+		tid:         tid,
+		host:        host,
+		c:           c,
+		id:          id,
+		bundle:      bundle,
+		isWCOW:      isWCOW,
+		spec:        spec,
+		io:          io,
+		processDone: make(chan struct{}),
+		state:       shimExecStateCreated,
+		exitStatus:  255, // By design for non-exited process status.
+		exited:      make(chan struct{}),
+		restore:     true,
+	}
+	p, err := cmd.RestoreCmd(ctx, filepath.Join(path, "cmd"), c, io)
+	if err != nil {
+		return nil, err
+	}
+	he.p = p
+	he.pid = he.p.Process.Pid()
+	he.state = shimExecStateRunning
+	go he.waitForContainerExit()
+	go he.waitForExit()
+	return he, nil
+}
+
+func (he *hcsExec) Save(ctx context.Context, path string) error {
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return err
+	}
+	if err := he.p.Save(ctx, filepath.Join(path, "cmd")); err != nil {
+		return err
+	}
+	return nil
+}
+
 var _ = (shimExec)(&hcsExec{})
 
 type hcsExec struct {
@@ -127,6 +187,7 @@ type hcsExec struct {
 	// exited is a wait block which waits async for the process to exit.
 	exited     chan struct{}
 	exitedOnce sync.Once
+	restore    bool
 }
 
 func (he *hcsExec) ID() string {
@@ -177,6 +238,9 @@ func (he *hcsExec) Status() *task.StateResponse {
 func (he *hcsExec) startInternal(ctx context.Context, initializeContainer bool) (err error) {
 	he.sl.Lock()
 	defer he.sl.Unlock()
+	if he.restore {
+		return nil
+	}
 	if he.state != shimExecStateCreated {
 		return newExecInvalidStateError(he.tid, he.id, he.state, "start")
 	}
@@ -185,7 +249,7 @@ func (he *hcsExec) startInternal(ctx context.Context, initializeContainer bool)
 			he.exitFromCreatedL(ctx, 1)
 		}
 	}()
-	if initializeContainer {
+	if !he.restore && initializeContainer {
 		err = he.c.Start(ctx)
 		if err != nil {
 			return err

cmd/containerd-shim-runhcs-v1/exec_wcow_podsandbox.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -206,3 +207,7 @@ func (wpse *wcowPodSandboxExec) ForceExit(ctx context.Context, status int) {
 		close(wpse.exited)
 	}
 }
+
+func (wpse *wcowPodSandboxExec) Save(ctx context.Context, path string) error {
+	return fmt.Errorf("not implemented")
+}

cmd/containerd-shim-runhcs-v1/pod.go

@@ -70,6 +70,8 @@ type shimPod interface {
 
 	StartSave(ctx context.Context, path string) error
 	CompleteSave(ctx context.Context, path string) error
+
+	RestoreTask(ctx context.Context, oldID string, scratchPath string, events publisher, req *task.CreateTaskRequest) (shimTask, error)
 }
 
 func createPod(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (_ shimPod, err error) {
@@ -299,6 +301,8 @@ type pod struct {
 	spec *specs.Spec
 
 	workloadTasks sync.Map
+
+	restorePath string
 }
 
 func (p *pod) ID() string {
@@ -472,6 +476,19 @@ func (p *pod) StartSave(ctx context.Context, path string) error {
 	if err := p.sandboxTask.Save(ctx, filepath.Join(path, "sandboxTask")); err != nil {
 		return err
 	}
+	var rangeErr error
+	p.workloadTasks.Range(func(key, value any) bool {
+		id := key.(string)
+		t := value.(shimTask)
+		if err := t.Save(ctx, filepath.Join(path, id)); err != nil {
+			rangeErr = err
+			return false
+		}
+		return true
+	})
+	if rangeErr != nil {
+		return rangeErr
+	}
 	if err := state.Write(filepath.Join(path, "state.json"),
 		&podState{
 			ID:   p.id,
@@ -500,12 +517,13 @@ type standbyPod struct {
 	host  *uvm.UtilityVM
 }
 
-func restorePod(ctx context.Context, path string, netNS string, scratchPath string, events publisher, req *task.CreateTaskRequest) (_ shimPod, err error) {
+func restorePod(ctx context.Context, path string, netNS string, resources map[string]string, events publisher, req *task.CreateTaskRequest) (_ shimPod, err error) {
 	p := &pod{
-		events: events,
-		id:     req.ID,
+		events:      events,
+		id:          req.ID,
+		restorePath: path,
 	}
-	p.host, err = uvm.RestoreUVM(ctx, filepath.Join(path, "uvm"), netNS, scratchPath, req.ID)
+	p.host, err = uvm.RestoreUVM(ctx, filepath.Join(path, "uvm"), netNS, resources, req.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -514,5 +532,19 @@ func restorePod(ctx context.Context, path string, netNS string, scratchPath stri
 		return nil, err
 	}
 	p.spec = state.Spec
+	st, err := restoreHcsTask(ctx, events, p.host, filepath.Join(path, "sandboxTask"), req, true, state.ID)
+	if err != nil {
+		return nil, err
+	}
+	p.sandboxTask = st
 	return p, nil
 }
+
+func (p *pod) RestoreTask(ctx context.Context, oldID string, scratchPath string, events publisher, req *task.CreateTaskRequest) (shimTask, error) {
+	t, err := restoreHcsTask(ctx, events, p.host, filepath.Join(p.restorePath, oldID), req, false, oldID)
+	if err != nil {
+		return nil, fmt.Errorf("restore task %s as %s: %w", oldID, req.ID, err)
+	}
+	p.workloadTasks.Store(req.ID, t)
+	return t, nil
+}

cmd/containerd-shim-runhcs-v1/serve.go

@@ -28,6 +28,7 @@ import (
 	hcslog "github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/save"
 	"github.com/Microsoft/hcsshim/internal/shimdiag"
+	"github.com/Microsoft/hcsshim/internal/winapi"
 	"github.com/Microsoft/hcsshim/pkg/octtrpc"
 )
 
@@ -48,6 +49,17 @@ var serveCommand = cli.Command{
 		},
 	},
 	Action: func(ctx *cli.Context) error {
+
+		_, err := os.Stat(`c:\debugwait`)
+		if err == nil {
+			for {
+				if winapi.IsDebuggerPresent() {
+					break
+				}
+				time.Sleep(1 * time.Second)
+			}
+		}
+
 		// On Windows the serve command is internally used to actually create
 		// the process that hosts the containerd/ttrpc entrypoint to the Runtime
 		// V2 API's. The model requires this 2nd invocation of the shim process

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -90,10 +90,11 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		return nil, err
 	}
 	var restoreSpec struct {
-		Typ         string
-		Path        string
-		NetNS       string
-		ScratchPath string
+		Typ       string
+		Path      string
+		ID        string
+		NetNS     string
+		Resources map[string]string
 	}
 	if err := json.Unmarshal(rawSpec, &restoreSpec); err != nil {
 		return nil, err
@@ -178,9 +179,14 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		if err == nil {
 			// The POD sandbox was previously created. Unlock and forward to the POD
 			s.cl.Unlock()
-			t, err := pod.CreateTask(ctx, req, &spec)
-			if err != nil {
-				return nil, err
+			var t shimTask
+			if restore {
+				t, err = pod.RestoreTask(ctx, restoreSpec.ID, "", s.events, req)
+			} else {
+				t, err = pod.CreateTask(ctx, req, &spec)
+				if err != nil {
+					return nil, err
+				}
 			}
 			e, _ := t.GetExec("")
 			resp.Pid = uint32(e.Pid())
@@ -191,7 +197,7 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 				ctx,
 				filepath.Join(restoreSpec.Path, "sandbox"),
 				restoreSpec.NetNS,
-				restoreSpec.ScratchPath,
+				restoreSpec.Resources,
 				s.events,
 				req,
 			)

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -44,6 +44,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
 	"github.com/Microsoft/hcsshim/internal/resources"
 	"github.com/Microsoft/hcsshim/internal/shimdiag"
+	"github.com/Microsoft/hcsshim/internal/state"
 	"github.com/Microsoft/hcsshim/internal/uvm"
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
@@ -1061,5 +1062,56 @@ func (ht *hcsTask) Save(ctx context.Context, path string) error {
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return err
 	}
+	if err := state.Write(filepath.Join(path, "config.json"), ht.taskSpec); err != nil {
+		return err
+	}
+	if err := ht.init.Save(ctx, filepath.Join(path, "init")); err != nil {
+		return err
+	}
 	return nil
 }
+
+func restoreHcsTask(ctx context.Context, events publisher, host *uvm.UtilityVM, path string, req *task.CreateTaskRequest, ownsParent bool, oldID string) (shimTask, error) {
+	io, err := cmd.NewUpstreamIO(ctx, req.ID, req.Stdout, req.Stderr, req.Stdin, req.Terminal, 0)
+	if err != nil {
+		return nil, err
+	}
+	container, err := host.RestoreContainer(ctx, oldID)
+	if err != nil {
+		return nil, err
+	}
+	spec, err := state.Read[specs.Spec](filepath.Join(path, "config.json"))
+	if err != nil {
+		return nil, err
+	}
+	ht := &hcsTask{
+		events:   events,
+		id:       req.ID,
+		c:        container,
+		ownsHost: ownsParent,
+		host:     host,
+		closed:   make(chan struct{}),
+		taskSpec: spec,
+		cr:       &resources.Resources{},
+	}
+	init, err := restoreHcsExec(
+		ctx,
+		events,
+		req.ID,
+		host,
+		container,
+		req.ID,
+		req.Bundle,
+		ht.isWCOW,
+		spec.Process,
+		io,
+		filepath.Join(path, "init"),
+	)
+	if err != nil {
+		return nil, err
+	}
+	ht.init = init
+	go ht.waitForHostExit()
+	go ht.waitInitExit()
+	return ht, nil
+}