From 8a2c56497edef6efe1544a6cff14a181ece7d0fe Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Sun, 28 Apr 2024 15:41:23 +0800 Subject: [PATCH] snapshot: fix error on proxy driver when switching different snapshotter During image pull, the containerd client calls Prepare API with the label containerd.io/snapshot.ref. When an image is pulled by other snapshotter, containerd doesn't send the label "containerd.io/snapshot.ref" to nydus snapshotter to let snapshotter prepare ro layers. Consequently, the code logic in nydus snapshotter cannot find label "containerd.io/snapshot/nydus-proxy-mode" and the logic of guest pull (proxy) is skipped. This occurs while reading the label data of parent snapshots(ro layers) during the preparation of the active snapshot(rw layer). Thus, when the snapshotter driver is configured to proxy mode, nydus snapshotter is compelled to implement the proxy logic. Fixes: #592 Signed-off-by: ChengyuZhu6 --- snapshot/process.go | 9 ++++++ snapshot/snapshot.go | 67 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/snapshot/process.go b/snapshot/process.go index 9f0da66f2d..c9bc224e90 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -57,6 +57,11 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, } } + proxyHandler := func() (bool, []mount.Mount, error) { + mounts, err := sn.mountProxy(ctx, s) + return false, mounts, err + } + // OCI image is also marked with "containerd.io/snapshot.ref" by Containerd target, isRoLayer := labels[label.TargetSnapshotRef] @@ -118,6 +123,10 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, // It should not be committed during this Prepare() operation. pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, sn.ms, parent) + if !treatAsProxyDriver(pInfo.Labels) { + logger.Warnf("treat as proxy mode for the prepared snapshot by other snapshotter possibly: id = %s, labels = %v", pID, pInfo.Labels) + handler = proxyHandler + } if pErr == nil && label.IsNydusProxyMode(pInfo.Labels) { logger.Infof("Prepare active snapshot %s in proxy mode", key) handler = remoteHandler(pID, pInfo.Labels) diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index f829d3132c..e341a60707 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -22,9 +22,9 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/continuity/fs" - "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/rafs" "github.com/containerd/nydus-snapshotter/pkg/cache" "github.com/containerd/nydus-snapshotter/pkg/cgroup" @@ -432,6 +432,11 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return nil, errors.Wrapf(err, "get snapshot %s", key) } + if !treatAsProxyDriver(info.Labels) { + log.L.Warnf("[Mounts] treat as proxy mode for the prepared snapshot by other snapshotter possibly: id = %s, labels = %v", id, info.Labels) + return o.mountProxy(ctx, *snap) + } + if needRemoteMounts { return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID, key) } @@ -838,6 +843,50 @@ func overlayMount(options []string) []mount.Mount { } } +// Handle proxy mount which the snapshot has been prepared by other snapshotter, mainly used for pause image in containerd +func (o *snapshotter) mountProxy(ctx context.Context, s storage.Snapshot) ([]mount.Mount, error) { + var overlayOptions []string + if s.Kind == snapshots.KindActive { + overlayOptions = append(overlayOptions, + fmt.Sprintf("workdir=%s", o.workPath(s.ID)), + fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), + ) + } + + log.G(ctx).Debugf("len(s.ParentIDs) = %v", len(s.ParentIDs)) + parentPaths := make([]string, 0, len(s.ParentIDs)+1) + if len(s.ParentIDs) == 0 { + parentPaths = append(parentPaths, config.GetSnapshotsRootDir()) + } else { + for _, id := range s.ParentIDs { + parentPaths = append(parentPaths, o.upperPath(id)) + } + } + + lowerDirOption := fmt.Sprintf("lowerdir=%s", strings.Join(parentPaths, ":")) + overlayOptions = append(overlayOptions, lowerDirOption) + log.G(ctx).Infof("proxy mount options %v", overlayOptions) + options, err := o.mountWithProxyVolume(rafs.Rafs{ + FsDriver: config.GetFsDriver(), + Annotations: make(map[string]string), + }) + if err != nil { + return []mount.Mount{}, errors.Wrapf(err, "create kata volume for proxy") + } + if len(options) > 0 { + overlayOptions = append(overlayOptions, options...) + } + log.G(ctx).Debugf("fuse.nydus-overlayfs mount options %v", overlayOptions) + mounts := []mount.Mount{ + { + Type: "fuse.nydus-overlayfs", + Source: "overlay", + Options: overlayOptions, + }, + } + return mounts, nil +} + // `s` is the upmost snapshot and `id` refers to the nydus meta snapshot // `s` and `id` can represent a different layer, it's useful when View an image func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id, key string) ([]mount.Mount, error) { @@ -1011,3 +1060,19 @@ func (o *snapshotter) snapshotRoot() string { func (o *snapshotter) snapshotDir(id string) string { return filepath.Join(o.snapshotRoot(), id) } + +func treatAsProxyDriver(labels map[string]string) bool { + isProxyDriver := config.GetFsDriver() == config.FsDriverProxy + isProxyLabel := label.IsNydusProxyMode(labels) + _, isProxyImage := labels[label.CRIImageRef] + log.G(context.Background()).Debugf("isProxyDriver = %t, isProxyLabel = %t, isProxyImage = %t", isProxyDriver, isProxyLabel, isProxyImage) + switch { + case isProxyDriver && isProxyImage: + return true + case isProxyDriver != isProxyLabel: + log.G(context.Background()).Warnf("check Labels With Driver failed, driver = %q, labels = %q", config.GetFsDriver(), labels) + return false + default: + return true + } +}