Store bootstrap parameters in sandbox metadata

api/next.pb.txt

@@ -5506,6 +5506,20 @@ file {
       type_name: ".containerd.services.sandbox.v1.ControllerStartResponse.LabelsEntry"
       json_name: "labels"
     }
+    field {
+      name: "address"
+      number: 5
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "address"
+    }
+    field {
+      name: "version"
+      number: 6
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT32
+      json_name: "version"
+    }
     nested_type {
       name: "LabelsEntry"
       field {
@@ -5696,6 +5710,20 @@ file {
       type_name: ".google.protobuf.Any"
       json_name: "extra"
     }
+    field {
+      name: "address"
+      number: 8
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "address"
+    }
+    field {
+      name: "version"
+      number: 9
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT32
+      json_name: "version"
+    }
     nested_type {
       name: "InfoEntry"
       field {

api/services/sandbox/v1/sandbox.proto

@@ -120,6 +120,11 @@ message ControllerStartResponse {
 	uint32 pid = 2;
 	google.protobuf.Timestamp created_at = 3;
 	map<string, string> labels = 4;
+	// Address of the sandbox for containerd to connect,
+	// for calling Task or other APIs serving in the sandbox.
+	// it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>.
+	string address = 5;
+	uint32 version = 6;
 }
 
 message ControllerPlatformRequest {
@@ -163,6 +168,11 @@ message ControllerStatusResponse {
 	google.protobuf.Timestamp created_at = 5;
 	google.protobuf.Timestamp exited_at = 6;
 	google.protobuf.Any extra = 7;
+	// Address of the sandbox for containerd to connect,
+	// for calling Task or other APIs serving in the sandbox.
+	// it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>.
+	string address = 8;
+	uint32 version = 9;
 }
 
 message ControllerShutdownRequest {

client/task_opts.go

@@ -50,6 +50,23 @@ func WithRuntimePath(absRuntimePath string) NewTaskOpts {
 	}
 }
 
+// WithTaskAPIEndpoint allow task service to manage a task through a given endpoint,
+// usually it is served inside a sandbox, and we can get it from sandbox status.
+func WithTaskAPIEndpoint(address string, version uint32) NewTaskOpts {
+	return func(ctx context.Context, client *Client, info *TaskInfo) error {
+		if info.Options == nil {
+			info.Options = &options.Options{}
+		}
+		opts, ok := info.Options.(*options.Options)
+		if !ok {
+			return errors.New("invalid runtime v2 options format")
+		}
+		opts.TaskApiAddress = address
+		opts.TaskApiVersion = version
+		return nil
+	}
+}
+
 // WithTaskCheckpoint allows a task to be created with live runtime and memory data from a
 // previous checkpoint. Additional software such as CRIU may be required to
 // restore a task from a checkpoint

core/runtime/runtime.go

@@ -51,6 +51,10 @@ type CreateOpts struct {
 	Runtime string
 	// SandboxID is an optional ID of sandbox this container belongs to
 	SandboxID string
+	// Address is an optional Address for Task API server
+	Address string
+	// Version is an optional Version of the Task API
+	Version uint32
 }
 
 // Exit information for a process

core/runtime/v2/binary.go

@@ -143,10 +143,12 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_
 	if err := writeBootstrapParams(filepath.Join(b.bundle.Path, "bootstrap.json"), params); err != nil {
 		return nil, fmt.Errorf("failed to write bootstrap.json: %w", err)
 	}
-
+	// The address is in the form like ttrpc+unix://<uds-path> or grpc+vsock://<cid>:<port>
+	address := fmt.Sprintf("%s+%s", params.Protocol, params.Address)
 	return &shim{
 		bundle:  b.bundle,
 		client:  conn,
+		address: address,
 		version: params.Version,
 	}, nil
 }

core/runtime/v2/runc/options/next.pb.txt

@@ -73,6 +73,20 @@ file {
       type: TYPE_STRING
       json_name: "criuWorkPath"
     }
+    field {
+      name: "task_api_address"
+      number: 12
+      label: LABEL_OPTIONAL
+      type: TYPE_STRING
+      json_name: "taskApiAddress"
+    }
+    field {
+      name: "task_api_version"
+      number: 13
+      label: LABEL_OPTIONAL
+      type: TYPE_UINT32
+      json_name: "taskApiVersion"
+    }
     reserved_range {
       start: 8
       end: 9

core/runtime/v2/runc/options/oci.proto

@@ -29,6 +29,11 @@ message Options {
 	string criu_image_path = 10;
 	// criu work path
 	string criu_work_path = 11;
+	// task api address, can be a unix domain socket, or vsock address.
+	// it is in the form of ttrpc+unix://path/to/uds or grpc+vsock://<vsock cid>:<port>.
+	string task_api_address = 12;
+	// task api version, currently supported value is 2 and 3.
+	uint32 task_api_version = 13;
 }
 
 message CheckpointOptions {

core/runtime/v2/shim.go

@@ -113,9 +113,13 @@ func loadShim(ctx context.Context, bundle *Bundle, onClose func()) (_ ShimInstan
 		}
 	}()
 
+	// The address is in the form like ttrpc+unix://<uds-path> or grpc+vsock://<cid>:<port>
+	address := fmt.Sprintf("%s+%s", params.Protocol, params.Address)
+
 	shim := &shim{
 		bundle:  bundle,
 		client:  conn,
+		address: address,
 		version: params.Version,
 	}
 
@@ -185,8 +189,9 @@ type ShimInstance interface {
 	Client() any
 	// Delete will close the client and remove bundle from disk.
 	Delete(ctx context.Context) error
-	// Version returns shim's features compatibility version.
-	Version() int
+	// Endpoint returns shim's endpoint information,
+	// including address and version.
+	Endpoint() (string, int)
 }
 
 func parseStartResponse(response []byte) (client.BootstrapParams, error) {
@@ -361,6 +366,7 @@ func (gc *grpcConn) UserOnCloseWait(ctx context.Context) error {
 type shim struct {
 	bundle  *Bundle
 	client  any
+	address string
 	version int
 }
 
@@ -371,8 +377,8 @@ func (s *shim) ID() string {
 	return s.bundle.ID
 }
 
-func (s *shim) Version() int {
-	return s.version
+func (s *shim) Endpoint() (string, int) {
+	return s.address, s.version
 }
 
 func (s *shim) Namespace() string {
@@ -440,7 +446,8 @@ type shimTask struct {
 }
 
 func newShimTask(shim ShimInstance) (*shimTask, error) {
-	taskClient, err := NewTaskClient(shim.Client(), shim.Version())
+	_, version := shim.Endpoint()
+	taskClient, err := NewTaskClient(shim.Client(), version)
 	if err != nil {
 		return nil, err
 	}

core/runtime/v2/shim_manager.go

@@ -163,21 +163,40 @@ func (m *ShimManager) ID() string {
 func (m *ShimManager) Start(ctx context.Context, id string, bundle *Bundle, opts runtime.CreateOpts) (_ ShimInstance, retErr error) {
 	// This container belongs to sandbox which supposed to be already started via sandbox API.
 	if opts.SandboxID != "" {
-		process, err := m.Get(ctx, opts.SandboxID)
-		if err != nil {
-			return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID)
+		var params shimbinary.BootstrapParams
+		if opts.Address != "" {
+			// The address returned from sandbox controller should be in the form like ttrpc+unix://<uds-path>
+			// or grpc+vsock://<cid>:<port>, we should get the protocol from the url first.
+			protocol, address, ok := strings.Cut(opts.Address, "+")
+			if !ok {
+				return nil, fmt.Errorf("the scheme of sandbox address should be in " +
+					" the form of <protocol>+<unix|vsock|tcp>, i.e. ttrpc+unix or grpc+vsock")
+			}
+			params = shimbinary.BootstrapParams{
+				Version:  int(opts.Version),
+				Protocol: protocol,
+				Address:  address,
+			}
+		} else {
+			// For those sandbox we can not get endpoint,
+			// fallback to legacy implementation
+			process, err := m.Get(ctx, opts.SandboxID)
+			if err != nil {
+				return nil, fmt.Errorf("can't find sandbox %s", opts.SandboxID)
+			}
+			p, restoreErr := restoreBootstrapParams(process.Bundle())
+			if restoreErr != nil {
+				return nil, fmt.Errorf("failed to get bootstrap "+
+					"params of sandbox %s, %v, legacy restore error %v", opts.SandboxID, err, restoreErr)
+			}
+			params = p
 		}
 
 		// Write sandbox ID this task belongs to.
 		if err := os.WriteFile(filepath.Join(bundle.Path, "sandbox"), []byte(opts.SandboxID), 0600); err != nil {
 			return nil, err
 		}
 
-		params, err := restoreBootstrapParams(process.Bundle())
-		if err != nil {
-			return nil, err
-		}
-
 		if err := writeBootstrapParams(filepath.Join(bundle.Path, "bootstrap.json"), params); err != nil {
 			return nil, fmt.Errorf("failed to write bootstrap.json for bundle %s: %w", bundle.Path, err)
 		}

core/sandbox/controller.go

@@ -117,6 +117,8 @@ type ControllerInstance struct {
 	SandboxID string
 	Pid       uint32
 	CreatedAt time.Time
+	Address   string
+	Version   uint32
 	Labels    map[string]string
 }
 
@@ -133,4 +135,6 @@ type ControllerStatus struct {
 	CreatedAt time.Time
 	ExitedAt  time.Time
 	Extra     typeurl.Any
+	Address   string
+	Version   uint32
 }

core/sandbox/proxy/controller.go

@@ -73,6 +73,8 @@ func (s *remoteSandboxController) Start(ctx context.Context, sandboxID string) (
 		Pid:       resp.GetPid(),
 		CreatedAt: resp.GetCreatedAt().AsTime(),
 		Labels:    resp.GetLabels(),
+		Address:   resp.GetAddress(),
+		Version:   resp.GetVersion(),
 	}, nil
 }
 
@@ -141,6 +143,8 @@ func (s *remoteSandboxController) Status(ctx context.Context, sandboxID string,
 		CreatedAt: resp.GetCreatedAt().AsTime(),
 		ExitedAt:  resp.GetExitedAt().AsTime(),
 		Extra:     resp.GetExtra(),
+		Address:   resp.GetAddress(),
+		Version:   resp.GetVersion(),
 	}, nil
 }