diff --git a/snapshot/process.go b/snapshot/process.go index 9f0da66f2d..3c20c0077d 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -57,6 +57,12 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, } } + proxyhandler := func() (bool, []mount.Mount, error) { + logger.Infof("Nydus proxy handler") + mounts, err := sn.mountProxy(ctx, labels, s) + return false, mounts, err + } + // OCI image is also marked with "containerd.io/snapshot.ref" by Containerd target, isRoLayer := labels[label.TargetSnapshotRef] @@ -118,6 +124,14 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, // It should not be committed during this Prepare() operation. pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, sn.ms, parent) + if checkErr := checkLabelsWithDriver(pInfo.Labels); checkErr != nil { + logger.Infof("not found proxy label: %v, err = %v", pInfo.Labels, checkErr) + // if pInfo.Labels == nil { + // pInfo.Labels = make(map[string]string) + // } + // pInfo.Labels[label.NydusProxyMode] = "true" + handler = proxyhandler + } if pErr == nil && label.IsNydusProxyMode(pInfo.Labels) { logger.Infof("Prepare active snapshot %s in proxy mode", key) handler = remoteHandler(pID, pInfo.Labels) diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index f829d3132c..8023c1e6d6 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -22,9 +22,9 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/continuity/fs" - "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + rafs "github.com/containerd/nydus-snapshotter/pkg/rafs" "github.com/containerd/nydus-snapshotter/pkg/cache" "github.com/containerd/nydus-snapshotter/pkg/cgroup" @@ -318,6 +318,7 @@ func (o *snapshotter) Cleanup(ctx context.Context) error { } func (o *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { + log.L.Debugf("[Stat] snapshots") _, info, _, err := snapshot.GetSnapshotInfo(ctx, o.ms, key) return info, err } @@ -432,6 +433,11 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return nil, errors.Wrapf(err, "get snapshot %s", key) } + if checkErr := checkLabelsWithDriver(info.Labels); checkErr != nil { + log.L.Infof("[Mounts] not found proxy label") + return o.mountProxy(ctx, info.Labels, *snap) + } + if needRemoteMounts { return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID, key) } @@ -838,6 +844,53 @@ func overlayMount(options []string) []mount.Mount { } } +func (o *snapshotter) mountProxy(ctx context.Context, labels map[string]string, s storage.Snapshot) ([]mount.Mount, error) { + var overlayOptions []string + var parentPaths []string + if s.Kind == snapshots.KindActive { + overlayOptions = append(overlayOptions, + fmt.Sprintf("workdir=%s", o.workPath(s.ID)), + fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), + ) + } + log.G(ctx).Infof("len(s.ParentIDs) = %v", len(s.ParentIDs)) + if len(s.ParentIDs) == 0 { + parentPaths = make([]string, 0, 8) + parentPaths = append(parentPaths, config.GetSnapshotsRootDir()) + } else { + parentPaths = make([]string, len(s.ParentIDs)) + for i := range s.ParentIDs { + parentPaths[i] = o.upperPath(s.ParentIDs[i]) + } + } + + lowerDirOption := fmt.Sprintf("lowerdir=%s", strings.Join(parentPaths, ":")) + overlayOptions = append(overlayOptions, lowerDirOption) + log.G(ctx).Infof("proxy mount options %v", overlayOptions) + options, err := o.mountWithProxyVolume(rafs.Rafs{ + FsDriver: config.GetFsDriver(), + ImageID: "", + SnapshotID: "", + SnapshotDir: "", + Annotations: make(map[string]string), + }) + if err != nil { + return []mount.Mount{}, errors.Wrapf(err, "create kata volume for proxy") + } + if len(options) > 0 { + overlayOptions = append(overlayOptions, options...) + } + log.G(ctx).Debugf("fuse.nydus-overlayfs mount options %v", overlayOptions) + mounts := []mount.Mount{ + { + Type: "fuse.nydus-overlayfs", + Source: "overlay", + Options: overlayOptions, + }, + } + return mounts, nil +} + // `s` is the upmost snapshot and `id` refers to the nydus meta snapshot // `s` and `id` can represent a different layer, it's useful when View an image func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id, key string) ([]mount.Mount, error) { @@ -1011,3 +1064,19 @@ func (o *snapshotter) snapshotRoot() string { func (o *snapshotter) snapshotDir(id string) string { return filepath.Join(o.snapshotRoot(), id) } + +func checkLabelsWithDriver(labels map[string]string) error { + isProxyDriver := config.GetFsDriver() == config.FsDriverProxy + isProxyLabel := label.IsNydusProxyMode(labels) + _, isProxyImage := labels[label.CRIImageRef] + log.G(context.Background()).Infof("isProxyDriver = %b", isProxyDriver) + log.G(context.Background()).Infof("isProxyLabel = %b", isProxyLabel) + log.G(context.Background()).Infof("isProxyImage = %b", isProxyImage) + if isProxyDriver && isProxyImage { + return nil + } + if (isProxyDriver && !isProxyLabel) || (!isProxyDriver && isProxyLabel) { + return errors.Errorf("check Labels With Driver failed, driver = %q, labels = %q", config.GetFsDriver(), labels) + } + return nil +}