Fix mount handling causing errors in VM

internal/mountutil/mount.go

@@ -0,0 +1,208 @@
+//go:build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// mountutil performs local mounts on Linux. This package should likely
+// be replaced with functions in the containerd mount code.
+package mountutil
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+	"time"
+
+	types "github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/log"
+)
+
+func All(ctx context.Context, rootfs, mdir string, mounts []*types.Mount) (retErr error) {
+	log.G(ctx).WithField("mounts", mounts).Debugf("mounting rootfs components")
+	active := []mount.ActiveMount{}
+
+	// TODO: Use mount manager interface, mount temps to directory
+	for i, m := range mounts {
+		var target string
+		if i < len(mounts)-1 {
+			target = filepath.Join(mdir, fmt.Sprintf("%d", i))
+			if err := os.MkdirAll(target, 0711); err != nil {
+				return err
+			}
+		} else {
+			target = rootfs
+		}
+		if t, ok := strings.CutPrefix(m.Type, "format/"); ok {
+			m.Type = t
+			for i, o := range m.Options {
+				format := formatString(o)
+				if format != nil {
+					s, err := format(active)
+					if err != nil {
+						return fmt.Errorf("formatting mount option %q: %w", o, err)
+					}
+					m.Options[i] = s
+				}
+			}
+			if format := formatString(m.Source); format != nil {
+				s, err := format(active)
+				if err != nil {
+					return fmt.Errorf("formatting mount source %q: %w", m.Source, err)
+				}
+				m.Source = s
+			}
+			if format := formatString(m.Target); format != nil {
+				s, err := format(active)
+				if err != nil {
+					return fmt.Errorf("formatting mount target %q: %w", m.Target, err)
+				}
+				m.Target = s
+			}
+		}
+		if t, ok := strings.CutPrefix(m.Type, "mkdir/"); ok {
+			m.Type = t
+			var options []string
+			for _, o := range m.Options {
+				if strings.HasPrefix(o, "X-containerd.mkdir.") {
+					prefix := "X-containerd.mkdir.path="
+					if !strings.HasPrefix(o, prefix) {
+						return fmt.Errorf("unknown mkdir mount option %q", o)
+					}
+					part := strings.SplitN(o[len(prefix):], ":", 4)
+					switch len(part) {
+					case 4:
+						// TODO: Support setting uid/gid
+						fallthrough
+					case 3:
+						fallthrough
+					case 2:
+						fallthrough
+					case 1:
+						dir := part[0]
+						if !strings.HasPrefix(dir, mdir) {
+							return fmt.Errorf("mkdir mount source %q must be under %q", dir, mdir)
+						}
+						if err := os.MkdirAll(dir, 0755); err != nil {
+							return err
+						}
+					default:
+						return fmt.Errorf("invalid mkdir mount option %q", o)
+					}
+				} else {
+					options = append(options, o)
+				}
+			}
+			m.Options = options
+
+		}
+		t := time.Now()
+		am := mount.ActiveMount{
+			Mount: mount.Mount{
+				Type:    m.Type,
+				Source:  m.Source,
+				Target:  m.Target,
+				Options: m.Options,
+			},
+			MountedAt:  &t,
+			MountPoint: target,
+		}
+		if err := am.Mount.Mount(target); err != nil {
+			return err
+		}
+		active = append(active, am)
+
+	}
+	defer func() {
+		if retErr != nil {
+			for i := len(active) - 1; i >= 0; i-- {
+				// TODO: delegate custom types to handlers
+				if active[i].Type == "mkdir" {
+					continue
+				}
+				if err := mount.UnmountAll(active[i].MountPoint, 0); err != nil {
+					log.G(ctx).WithError(err).WithField("mountpoint", active[i].MountPoint).Warn("failed to cleanup mount")
+				}
+			}
+		}
+	}()
+
+	return nil
+}
+
+const formatCheck = "{{"
+
+func formatString(s string) func([]mount.ActiveMount) (string, error) {
+	if !strings.Contains(s, formatCheck) {
+		return nil
+	}
+
+	return func(a []mount.ActiveMount) (string, error) {
+		fm := template.FuncMap{
+			"source": func(i int) (string, error) {
+				if i < 0 || i >= len(a) {
+					return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
+				}
+				return a[i].Source, nil
+			},
+			"target": func(i int) (string, error) {
+				if i < 0 || i >= len(a) {
+					return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
+				}
+				return a[i].Target, nil
+			},
+			"mount": func(i int) (string, error) {
+				if i < 0 || i >= len(a) {
+					return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
+				}
+				return a[i].MountPoint, nil
+			},
+			"overlay": func(start, end int) (string, error) {
+				var dirs []string
+				if start > end {
+					if start >= len(a) || end < 0 {
+						return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a))
+					}
+					for i := start; i >= end; i-- {
+						dirs = append(dirs, a[i].MountPoint)
+					}
+				} else {
+					if start < 0 || end >= len(a) {
+						return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a))
+					}
+					for i := start; i <= end; i++ {
+						dirs = append(dirs, a[i].MountPoint)
+					}
+				}
+				return strings.Join(dirs, ":"), nil
+			},
+		}
+		t, err := template.New("").Funcs(fm).Parse(s)
+		if err != nil {
+			return "", err
+		}
+
+		buf := bytes.NewBuffer(nil)
+		if err := t.Execute(buf, nil); err != nil {
+			return "", err
+		}
+		return buf.String(), nil
+	}
+}

internal/shim/manager/manager.go

@@ -333,7 +333,7 @@ func (m manager) Info(ctx context.Context, optionsR io.Reader) (*types.RuntimeIn
 			//Revision: version.Revision,
 		},
 		Annotations: map[string]string{
-			"containerd.io/runtime-allow-mounts": "mkdir/*,format/*",
+			"containerd.io/runtime-allow-mounts": "mkdir/*,format/*,erofs",
 		},
 	}
 	// TODO: Get features list from run_vminitd

internal/shim/task/mount.go

@@ -19,21 +19,29 @@ package task
 import (
 	"context"
 	"fmt"
-	"path/filepath"
 	"strings"
 
 	"github.com/containerd/containerd/api/types"
+	"github.com/containerd/errdefs"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdbox/internal/vm"
 )
 
+type diskOptions struct {
+	name     string
+	source   string
+	readOnly bool
+}
+
 // transformMounts does not perform any local mounts but transforms
 // the mounts to be used inside the VM via virtio
 func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount) ([]*types.Mount, error) {
 	var (
-		disks byte = 'a'
-		am    []*types.Mount
+		disks    byte = 'a'
+		addDisks []diskOptions
+		am       []*types.Mount
+		err      error
 	)
 
 	for _, m := range ms {
@@ -44,9 +52,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
 			if len(disk) > 36 {
 				disk = disk[:36]
 			}
-			if err := vmi.AddDisk(ctx, disk, m.Source, vm.WithReadOnly()); err != nil {
-				return nil, err
-			}
+			addDisks = append(addDisks, diskOptions{
+				name:     disk,
+				source:   m.Source,
+				readOnly: true,
+			})
 			am = append(am, &types.Mount{
 				Type:    "erofs",
 				Source:  fmt.Sprintf("/dev/vd%c", disks),
@@ -60,9 +70,12 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
 			if len(disk) > 36 {
 				disk = disk[:36]
 			}
-			if err := vmi.AddDisk(ctx, disk, m.Source); err != nil {
-				return nil, err
-			}
+			// TODO: Check read only option
+			addDisks = append(addDisks, diskOptions{
+				name:     disk,
+				source:   m.Source,
+				readOnly: false,
+			})
 			am = append(am, &types.Mount{
 				Type:    "ext4",
 				Source:  fmt.Sprintf("/dev/vd%c", disks),
@@ -87,35 +100,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
 				// TODO: Handle virtio for lowers?
 			}
 			if wdi > -1 && udi > -1 {
-				udir, uname := filepath.Split(m.Options[udi][len("upperdir="):])
-
-				wdir, wname := filepath.Split(m.Options[wdi][len("workdir="):])
-				if udir == wdir {
-					tag := fmt.Sprintf("overlayfs-upper-%s", id)
-					// virtiofs implementation has a limit of 36 characters for the tag
-					if len(tag) > 36 {
-						tag = tag[:36]
-					}
-					if err := vmi.AddFS(ctx, tag, udir); err != nil {
-						return nil, err
-					}
-					m.Options[udi] = fmt.Sprintf("upperdir={{ mount %d }}/%s", len(am), uname)
-					m.Options[wdi] = fmt.Sprintf("workdir={{ mount %d }}/%s", len(am), wname)
-					am = append(am, &types.Mount{
-						Type:   "virtiofs",
-						Source: tag,
-					})
-					log.G(ctx).WithFields(log.Fields{
-						"workdir":  m.Options[wdi],
-						"upperdir": m.Options[udi],
-					}).Warnf("transformed upper and work")
-				} else {
-					log.G(ctx).WithFields(log.Fields{
-						"workdir":  m.Options[wdi],
-						"upperdir": m.Options[udi],
-					}).Warnf("overlayfs workdir and upperdir should be in the same directory")
-				}
-			} else {
+				// Having the upper as virtiofs may return invalid argument, avoid
+				// transforming and attempt to perform the mounts on the host if
+				// supported.
+				return nil, fmt.Errorf("cannot use virtiofs for upper dir in overlay: %w", errdefs.ErrNotImplemented)
+			} else if wdi == -1 || udi == -1 {
 				log.G(ctx).WithField("options", m.Options).Warnf("overlayfs missing workdir or upperdir")
 			}
 
@@ -125,7 +114,21 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
 		}
 	}
 
-	return am, nil
+	if len(addDisks) > 10 {
+		return nil, fmt.Errorf("exceeded maximum virtio disk count: %d > 10: %w", len(addDisks), errdefs.ErrNotImplemented)
+	}
+
+	for _, do := range addDisks {
+		var opts []vm.MountOpt
+		if do.readOnly {
+			opts = append(opts, vm.WithReadOnly())
+		}
+		if err := vmi.AddDisk(ctx, do.name, do.source, opts...); err != nil {
+			return nil, err
+		}
+	}
+
+	return am, err
 }
 
 func filterOptions(options []string) []string {

internal/shim/task/mount_darwin.go

@@ -24,6 +24,6 @@ import (
 	"github.com/containerd/nerdbox/internal/vm"
 )
 
-func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _ string) ([]*types.Mount, error) {
+func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _, _ string) ([]*types.Mount, error) {
 	return transformMounts(ctx, vmi, id, ms)
 }

internal/shim/task/mount_linux.go

@@ -22,11 +22,13 @@ import (
 
 	"github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/errdefs"
 
+	"github.com/containerd/nerdbox/internal/mountutil"
 	"github.com/containerd/nerdbox/internal/vm"
 )
 
-func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs string) ([]*types.Mount, error) {
+func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs, lmounts string) ([]*types.Mount, error) {
 	// Handle mounts
 
 	if len(m) == 1 && (m[0].Type == "overlay" || m[0].Type == "bind") {
@@ -66,5 +68,25 @@ func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mou
 			Source: tag,
 		}}, nil
 	}
-	return transformMounts(ctx, vmi, id, m)
+	mounts, err := transformMounts(ctx, vmi, id, m)
+	if err != nil && errdefs.IsNotImplemented(err) {
+		if err := mountutil.All(ctx, rootfs, lmounts, m); err != nil {
+			return nil, err
+		}
+
+		// Fallback to original rootfs mount
+		tag := fmt.Sprintf("rootfs-%s", id)
+		// virtiofs implementation has a limit of 36 characters for the tag
+		if len(tag) > 36 {
+			tag = tag[:36]
+		}
+		if err := vmi.AddFS(ctx, tag, rootfs); err != nil {
+			return nil, err
+		}
+		return []*types.Mount{{
+			Type:   "virtiofs",
+			Source: tag,
+		}}, nil
+	}
+	return mounts, err
 }