diff --git a/internal/mountutil/mount.go b/internal/mountutil/mount.go new file mode 100644 index 0000000..900ce21 --- /dev/null +++ b/internal/mountutil/mount.go @@ -0,0 +1,208 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// mountutil performs local mounts on Linux. This package should likely +// be replaced with functions in the containerd mount code. +package mountutil + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + "time" + + types "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/log" +) + +func All(ctx context.Context, rootfs, mdir string, mounts []*types.Mount) (retErr error) { + log.G(ctx).WithField("mounts", mounts).Debugf("mounting rootfs components") + active := []mount.ActiveMount{} + + // TODO: Use mount manager interface, mount temps to directory + for i, m := range mounts { + var target string + if i < len(mounts)-1 { + target = filepath.Join(mdir, fmt.Sprintf("%d", i)) + if err := os.MkdirAll(target, 0711); err != nil { + return err + } + } else { + target = rootfs + } + if t, ok := strings.CutPrefix(m.Type, "format/"); ok { + m.Type = t + for i, o := range m.Options { + format := formatString(o) + if format != nil { + s, err := format(active) + if err != nil { + return fmt.Errorf("formatting mount option %q: %w", o, err) + } + m.Options[i] = s + } + } + if format := formatString(m.Source); format != nil { + s, err := format(active) + if err != nil { + return fmt.Errorf("formatting mount source %q: %w", m.Source, err) + } + m.Source = s + } + if format := formatString(m.Target); format != nil { + s, err := format(active) + if err != nil { + return fmt.Errorf("formatting mount target %q: %w", m.Target, err) + } + m.Target = s + } + } + if t, ok := strings.CutPrefix(m.Type, "mkdir/"); ok { + m.Type = t + var options []string + for _, o := range m.Options { + if strings.HasPrefix(o, "X-containerd.mkdir.") { + prefix := "X-containerd.mkdir.path=" + if !strings.HasPrefix(o, prefix) { + return fmt.Errorf("unknown mkdir mount option %q", o) + } + part := strings.SplitN(o[len(prefix):], ":", 4) + switch len(part) { + case 4: + // TODO: Support setting uid/gid + fallthrough + case 3: + fallthrough + case 2: + fallthrough + case 1: + dir := part[0] + if !strings.HasPrefix(dir, mdir) { + return fmt.Errorf("mkdir mount source %q must be under %q", dir, mdir) + } + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + default: + return fmt.Errorf("invalid mkdir mount option %q", o) + } + } else { + options = append(options, o) + } + } + m.Options = options + + } + t := time.Now() + am := mount.ActiveMount{ + Mount: mount.Mount{ + Type: m.Type, + Source: m.Source, + Target: m.Target, + Options: m.Options, + }, + MountedAt: &t, + MountPoint: target, + } + if err := am.Mount.Mount(target); err != nil { + return err + } + active = append(active, am) + + } + defer func() { + if retErr != nil { + for i := len(active) - 1; i >= 0; i-- { + // TODO: delegate custom types to handlers + if active[i].Type == "mkdir" { + continue + } + if err := mount.UnmountAll(active[i].MountPoint, 0); err != nil { + log.G(ctx).WithError(err).WithField("mountpoint", active[i].MountPoint).Warn("failed to cleanup mount") + } + } + } + }() + + return nil +} + +const formatCheck = "{{" + +func formatString(s string) func([]mount.ActiveMount) (string, error) { + if !strings.Contains(s, formatCheck) { + return nil + } + + return func(a []mount.ActiveMount) (string, error) { + fm := template.FuncMap{ + "source": func(i int) (string, error) { + if i < 0 || i >= len(a) { + return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) + } + return a[i].Source, nil + }, + "target": func(i int) (string, error) { + if i < 0 || i >= len(a) { + return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) + } + return a[i].Target, nil + }, + "mount": func(i int) (string, error) { + if i < 0 || i >= len(a) { + return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) + } + return a[i].MountPoint, nil + }, + "overlay": func(start, end int) (string, error) { + var dirs []string + if start > end { + if start >= len(a) || end < 0 { + return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a)) + } + for i := start; i >= end; i-- { + dirs = append(dirs, a[i].MountPoint) + } + } else { + if start < 0 || end >= len(a) { + return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a)) + } + for i := start; i <= end; i++ { + dirs = append(dirs, a[i].MountPoint) + } + } + return strings.Join(dirs, ":"), nil + }, + } + t, err := template.New("").Funcs(fm).Parse(s) + if err != nil { + return "", err + } + + buf := bytes.NewBuffer(nil) + if err := t.Execute(buf, nil); err != nil { + return "", err + } + return buf.String(), nil + } +} diff --git a/internal/shim/manager/manager.go b/internal/shim/manager/manager.go index 4831157..e9b1d98 100644 --- a/internal/shim/manager/manager.go +++ b/internal/shim/manager/manager.go @@ -333,7 +333,7 @@ func (m manager) Info(ctx context.Context, optionsR io.Reader) (*types.RuntimeIn //Revision: version.Revision, }, Annotations: map[string]string{ - "containerd.io/runtime-allow-mounts": "mkdir/*,format/*", + "containerd.io/runtime-allow-mounts": "mkdir/*,format/*,erofs", }, } // TODO: Get features list from run_vminitd diff --git a/internal/shim/task/mount.go b/internal/shim/task/mount.go index 87ddd4f..81fb7de 100644 --- a/internal/shim/task/mount.go +++ b/internal/shim/task/mount.go @@ -19,21 +19,29 @@ package task import ( "context" "fmt" - "path/filepath" "strings" "github.com/containerd/containerd/api/types" + "github.com/containerd/errdefs" "github.com/containerd/log" "github.com/containerd/nerdbox/internal/vm" ) +type diskOptions struct { + name string + source string + readOnly bool +} + // transformMounts does not perform any local mounts but transforms // the mounts to be used inside the VM via virtio func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount) ([]*types.Mount, error) { var ( - disks byte = 'a' - am []*types.Mount + disks byte = 'a' + addDisks []diskOptions + am []*types.Mount + err error ) for _, m := range ms { @@ -44,9 +52,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type if len(disk) > 36 { disk = disk[:36] } - if err := vmi.AddDisk(ctx, disk, m.Source, vm.WithReadOnly()); err != nil { - return nil, err - } + addDisks = append(addDisks, diskOptions{ + name: disk, + source: m.Source, + readOnly: true, + }) am = append(am, &types.Mount{ Type: "erofs", Source: fmt.Sprintf("/dev/vd%c", disks), @@ -60,9 +70,12 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type if len(disk) > 36 { disk = disk[:36] } - if err := vmi.AddDisk(ctx, disk, m.Source); err != nil { - return nil, err - } + // TODO: Check read only option + addDisks = append(addDisks, diskOptions{ + name: disk, + source: m.Source, + readOnly: false, + }) am = append(am, &types.Mount{ Type: "ext4", Source: fmt.Sprintf("/dev/vd%c", disks), @@ -87,35 +100,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type // TODO: Handle virtio for lowers? } if wdi > -1 && udi > -1 { - udir, uname := filepath.Split(m.Options[udi][len("upperdir="):]) - - wdir, wname := filepath.Split(m.Options[wdi][len("workdir="):]) - if udir == wdir { - tag := fmt.Sprintf("overlayfs-upper-%s", id) - // virtiofs implementation has a limit of 36 characters for the tag - if len(tag) > 36 { - tag = tag[:36] - } - if err := vmi.AddFS(ctx, tag, udir); err != nil { - return nil, err - } - m.Options[udi] = fmt.Sprintf("upperdir={{ mount %d }}/%s", len(am), uname) - m.Options[wdi] = fmt.Sprintf("workdir={{ mount %d }}/%s", len(am), wname) - am = append(am, &types.Mount{ - Type: "virtiofs", - Source: tag, - }) - log.G(ctx).WithFields(log.Fields{ - "workdir": m.Options[wdi], - "upperdir": m.Options[udi], - }).Warnf("transformed upper and work") - } else { - log.G(ctx).WithFields(log.Fields{ - "workdir": m.Options[wdi], - "upperdir": m.Options[udi], - }).Warnf("overlayfs workdir and upperdir should be in the same directory") - } - } else { + // Having the upper as virtiofs may return invalid argument, avoid + // transforming and attempt to perform the mounts on the host if + // supported. + return nil, fmt.Errorf("cannot use virtiofs for upper dir in overlay: %w", errdefs.ErrNotImplemented) + } else if wdi == -1 || udi == -1 { log.G(ctx).WithField("options", m.Options).Warnf("overlayfs missing workdir or upperdir") } @@ -125,7 +114,21 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type } } - return am, nil + if len(addDisks) > 10 { + return nil, fmt.Errorf("exceeded maximum virtio disk count: %d > 10: %w", len(addDisks), errdefs.ErrNotImplemented) + } + + for _, do := range addDisks { + var opts []vm.MountOpt + if do.readOnly { + opts = append(opts, vm.WithReadOnly()) + } + if err := vmi.AddDisk(ctx, do.name, do.source, opts...); err != nil { + return nil, err + } + } + + return am, err } func filterOptions(options []string) []string { diff --git a/internal/shim/task/mount_darwin.go b/internal/shim/task/mount_darwin.go index ceb5c26..fa30fff 100644 --- a/internal/shim/task/mount_darwin.go +++ b/internal/shim/task/mount_darwin.go @@ -24,6 +24,6 @@ import ( "github.com/containerd/nerdbox/internal/vm" ) -func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _ string) ([]*types.Mount, error) { +func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _, _ string) ([]*types.Mount, error) { return transformMounts(ctx, vmi, id, ms) } diff --git a/internal/shim/task/mount_linux.go b/internal/shim/task/mount_linux.go index 5061def..a697509 100644 --- a/internal/shim/task/mount_linux.go +++ b/internal/shim/task/mount_linux.go @@ -22,11 +22,13 @@ import ( "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/errdefs" + "github.com/containerd/nerdbox/internal/mountutil" "github.com/containerd/nerdbox/internal/vm" ) -func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs string) ([]*types.Mount, error) { +func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs, lmounts string) ([]*types.Mount, error) { // Handle mounts if len(m) == 1 && (m[0].Type == "overlay" || m[0].Type == "bind") { @@ -66,5 +68,25 @@ func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mou Source: tag, }}, nil } - return transformMounts(ctx, vmi, id, m) + mounts, err := transformMounts(ctx, vmi, id, m) + if err != nil && errdefs.IsNotImplemented(err) { + if err := mountutil.All(ctx, rootfs, lmounts, m); err != nil { + return nil, err + } + + // Fallback to original rootfs mount + tag := fmt.Sprintf("rootfs-%s", id) + // virtiofs implementation has a limit of 36 characters for the tag + if len(tag) > 36 { + tag = tag[:36] + } + if err := vmi.AddFS(ctx, tag, rootfs); err != nil { + return nil, err + } + return []*types.Mount{{ + Type: "virtiofs", + Source: tag, + }}, nil + } + return mounts, err } diff --git a/internal/shim/task/mount_other.go b/internal/shim/task/mount_other.go index 56eb731..9a7441e 100644 --- a/internal/shim/task/mount_other.go +++ b/internal/shim/task/mount_other.go @@ -26,6 +26,6 @@ import ( "github.com/containerd/nerdbox/internal/vm" ) -func setupMounts(_ context.Context, vmi vm.Instance, id string, ms []*types.Mount, rootfs string) ([]*types.Mount, error) { +func setupMounts(_ context.Context, vmi vm.Instance, id string, ms []*types.Mount, rootfs, _ string) ([]*types.Mount, error) { return ms, nil } diff --git a/internal/shim/task/service.go b/internal/shim/task/service.go index 9f6c261..6f041f4 100644 --- a/internal/shim/task/service.go +++ b/internal/shim/task/service.go @@ -216,7 +216,7 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ * return nil, errgrpc.ToGRPC(err) } - m, err := setupMounts(ctx, vmi, r.ID, r.Rootfs, b.Rootfs) + m, err := setupMounts(ctx, vmi, r.ID, r.Rootfs, b.Rootfs, filepath.Join(r.Bundle, "mounts")) if err != nil { return nil, errgrpc.ToGRPC(err) } diff --git a/internal/vminit/runc/container.go b/internal/vminit/runc/container.go index 56cf4f8..0df5aeb 100644 --- a/internal/vminit/runc/container.go +++ b/internal/vminit/runc/container.go @@ -19,16 +19,12 @@ package runc import ( - "bytes" "context" "encoding/json" "fmt" "os" "path/filepath" - "strings" "sync" - "text/template" - "time" "github.com/containerd/cgroups/v3" "github.com/containerd/cgroups/v3/cgroup1" @@ -36,12 +32,12 @@ import ( "github.com/containerd/console" "github.com/containerd/containerd/api/runtime/task/v3" "github.com/containerd/containerd/api/types/runc/options" - "github.com/containerd/containerd/v2/core/mount" "github.com/containerd/containerd/v2/pkg/stdio" "github.com/containerd/errdefs" "github.com/containerd/log" "github.com/containerd/typeurl/v2" + "github.com/containerd/nerdbox/internal/mountutil" "github.com/containerd/nerdbox/internal/vminit/process" "github.com/containerd/nerdbox/internal/vminit/stream" ) @@ -97,119 +93,12 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa return nil, err } - var mounts []mount.Mount - for _, pm := range pmounts { - mounts = append(mounts, mount.Mount{ - Type: pm.Type, - Source: pm.Source, - Target: pm.Target, - Options: pm.Options, - }) - } - - if len(mounts) != 0 && (len(mounts) != 1 || mounts[0].Type != "bind" || mounts[0].Source != rootfs) { - log.G(ctx).WithField("mounts", mounts).Debugf("mounting rootfs components") + if len(r.Rootfs) != 0 && (len(r.Rootfs) != 1 || r.Rootfs[0].Type != "bind" || r.Rootfs[0].Source != rootfs) { + log.G(ctx).WithField("mounts", r.Rootfs).Debugf("mounting rootfs components") mdir := filepath.Join(r.Bundle, "mounts") - active := []mount.ActiveMount{} - - // TODO: Use mount manager interface, mount temps to directory - for i, m := range mounts { - var target string - if i < len(mounts)-1 { - target = filepath.Join(mdir, fmt.Sprintf("%d", i)) - if err := os.MkdirAll(target, 0711); err != nil { - return nil, err - } - } else { - target = rootfs - } - if t, ok := strings.CutPrefix(m.Type, "format/"); ok { - m.Type = t - for i, o := range m.Options { - format := formatString(o) - if format != nil { - s, err := format(active) - if err != nil { - return nil, fmt.Errorf("formatting mount option %q: %w", o, err) - } - m.Options[i] = s - } - } - if format := formatString(m.Source); format != nil { - s, err := format(active) - if err != nil { - return nil, fmt.Errorf("formatting mount source %q: %w", m.Source, err) - } - m.Source = s - } - if format := formatString(m.Target); format != nil { - s, err := format(active) - if err != nil { - return nil, fmt.Errorf("formatting mount target %q: %w", m.Target, err) - } - m.Target = s - } - } - if t, ok := strings.CutPrefix(m.Type, "mkdir/"); ok { - m.Type = t - var options []string - for _, o := range m.Options { - if strings.HasPrefix(o, "X-containerd.mkdir.") { - prefix := "X-containerd.mkdir.path=" - if !strings.HasPrefix(o, prefix) { - return nil, fmt.Errorf("unknown mkdir mount option %q", o) - } - part := strings.SplitN(o[len(prefix):], ":", 4) - switch len(part) { - case 4: - // TODO: Support setting uid/gid - fallthrough - case 3: - fallthrough - case 2: - fallthrough - case 1: - dir := part[0] - if !strings.HasPrefix(dir, mdir) { - return nil, fmt.Errorf("mkdir mount source %q must be under %q", dir, mdir) - } - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("invalid mkdir mount option %q", o) - } - } else { - options = append(options, o) - } - } - m.Options = options - - } - if err := m.Mount(target); err != nil { - return nil, err - } - t := time.Now() - active = append(active, mount.ActiveMount{ - Mount: m, - MountedAt: &t, - MountPoint: target, - }) - + if err := mountutil.All(ctx, rootfs, mdir, r.Rootfs); err != nil { + return nil, err } - defer func() { - if retErr != nil { - for i := len(active) - 1; i >= 0; i-- { - // TODO: delegate custom types to handlers - if active[i].Type == "mkdir" { - continue - } - if err := mount.UnmountAll(active[i].MountPoint, 0); err != nil { - log.G(ctx).WithError(err).WithField("mountpoint", active[i].MountPoint).Warn("failed to cleanup mount") - } - } - } - }() } p, err := newInit( @@ -244,66 +133,6 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa return container, nil } -const formatCheck = "{{" - -func formatString(s string) func([]mount.ActiveMount) (string, error) { - if !strings.Contains(s, formatCheck) { - return nil - } - - return func(a []mount.ActiveMount) (string, error) { - fm := template.FuncMap{ - "source": func(i int) (string, error) { - if i < 0 || i >= len(a) { - return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) - } - return a[i].Source, nil - }, - "target": func(i int) (string, error) { - if i < 0 || i >= len(a) { - return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) - } - return a[i].Target, nil - }, - "mount": func(i int) (string, error) { - if i < 0 || i >= len(a) { - return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a)) - } - return a[i].MountPoint, nil - }, - "overlay": func(start, end int) (string, error) { - var dirs []string - if start > end { - if start >= len(a) || end < 0 { - return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a)) - } - for i := start; i >= end; i-- { - dirs = append(dirs, a[i].MountPoint) - } - } else { - if start < 0 || end >= len(a) { - return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a)) - } - for i := start; i <= end; i++ { - dirs = append(dirs, a[i].MountPoint) - } - } - return strings.Join(dirs, ":"), nil - }, - } - t, err := template.New("").Funcs(fm).Parse(s) - if err != nil { - return "", err - } - - buf := bytes.NewBuffer(nil) - if err := t.Execute(buf, nil); err != nil { - return "", err - } - return buf.String(), nil - } -} - const optionsFilename = "options.json" // ReadOptions reads the option information from the path.