snapshot: fix error on proxy driver when switching different snapshotter

During image pull, the containerd client calls Prepare API with the label containerd.io/snapshot.ref.
When an image is pulled by other snapshotter, containerd doesn't send the label "containerd.io/snapshot.ref" to
nydus snapshotter to let snapshotter prepare ro layers.
Consequently, the code logic in nydus snapshotter cannot find label "containerd.io/snapshot/nydus-proxy-mode"
and the logic of guest pull (proxy) is skipped.
This occurs while reading the label data of parent snapshots(ro layers) during the preparation of the active snapshot(rw layer).
Thus, when the snapshotter driver is configured to proxy mode, nydus snapshotter is compelled to implement the proxy logic.

Fixes: #592

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>

snapshot/process.go

@@ -57,6 +57,11 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry,
 		}
 	}
 
+	proxyHandler := func() (bool, []mount.Mount, error) {
+		mounts, err := sn.mountProxy(ctx, s)
+		return false, mounts, err
+	}
+
 	// OCI image is also marked with "containerd.io/snapshot.ref" by Containerd
 	target, isRoLayer := labels[label.TargetSnapshotRef]
 
@@ -118,6 +123,10 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry,
 		// It should not be committed during this Prepare() operation.
 
 		pID, pInfo, _, pErr := snapshot.GetSnapshotInfo(ctx, sn.ms, parent)
+		if checkErr := treatAsProxyDriver(pInfo.Labels); checkErr != nil {
+			logger.Warnf("not found proxy label: %v, err = %v", pInfo.Labels, checkErr)
+			handler = proxyHandler
+		}
 		if pErr == nil && label.IsNydusProxyMode(pInfo.Labels) {
 			logger.Infof("Prepare active snapshot %s in proxy mode", key)
 			handler = remoteHandler(pID, pInfo.Labels)

snapshot/snapshot.go

@@ -22,9 +22,9 @@ import (
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/storage"
 	"github.com/containerd/continuity/fs"
-
 	"github.com/containerd/nydus-snapshotter/config"
 	"github.com/containerd/nydus-snapshotter/config/daemonconfig"
+	"github.com/containerd/nydus-snapshotter/pkg/rafs"
 
 	"github.com/containerd/nydus-snapshotter/pkg/cache"
 	"github.com/containerd/nydus-snapshotter/pkg/cgroup"
@@ -432,6 +432,11 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er
 		return nil, errors.Wrapf(err, "get snapshot %s", key)
 	}
 
+	if checkErr := treatAsProxyDriver(info.Labels); checkErr != nil {
+		log.L.Debugf("[Mounts] not found proxy label, err = %v", checkErr)
+		return o.mountProxy(ctx, *snap)
+	}
+
 	if needRemoteMounts {
 		return o.mountRemote(ctx, info.Labels, *snap, metaSnapshotID, key)
 	}
@@ -838,6 +843,49 @@ func overlayMount(options []string) []mount.Mount {
 	}
 }
 
+// Handle proxy mount which the snapshot has been prepared by other snapshotter, mainly used for pause image in containerd
+func (o *snapshotter) mountProxy(ctx context.Context, s storage.Snapshot) ([]mount.Mount, error) {
+	var overlayOptions []string
+	if s.Kind == snapshots.KindActive {
+		overlayOptions = append(overlayOptions,
+			fmt.Sprintf("workdir=%s", o.workPath(s.ID)),
+			fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)),
+		)
+	}
+
+	log.G(ctx).Debugf("len(s.ParentIDs) = %v", len(s.ParentIDs))
+	parentPaths := make([]string, 0, len(s.ParentIDs)+1)
+	if len(s.ParentIDs) == 0 {
+		parentPaths = append(parentPaths, config.GetSnapshotsRootDir())
+	} else {
+		for _, id := range s.ParentIDs {
+			parentPaths = append(parentPaths, o.upperPath(id))
+		}
+	}
+
+	lowerDirOption := fmt.Sprintf("lowerdir=%s", strings.Join(parentPaths, ":"))
+	overlayOptions = append(overlayOptions, lowerDirOption)
+	log.G(ctx).Infof("proxy mount options %v", overlayOptions)
+	options, err := o.mountWithProxyVolume(rafs.Rafs{
+		FsDriver: config.GetFsDriver(),
+	})
+	if err != nil {
+		return []mount.Mount{}, errors.Wrapf(err, "create kata volume for proxy")
+	}
+	if len(options) > 0 {
+		overlayOptions = append(overlayOptions, options...)
+	}
+	log.G(ctx).Debugf("fuse.nydus-overlayfs mount options %v", overlayOptions)
+	mounts := []mount.Mount{
+		{
+			Type:    "fuse.nydus-overlayfs",
+			Source:  "overlay",
+			Options: overlayOptions,
+		},
+	}
+	return mounts, nil
+}
+
 // `s` is the upmost snapshot and `id` refers to the nydus meta snapshot
 // `s` and `id` can represent a different layer, it's useful when View an image
 func (o *snapshotter) mountRemote(ctx context.Context, labels map[string]string, s storage.Snapshot, id, key string) ([]mount.Mount, error) {
@@ -1011,3 +1059,18 @@ func (o *snapshotter) snapshotRoot() string {
 func (o *snapshotter) snapshotDir(id string) string {
 	return filepath.Join(o.snapshotRoot(), id)
 }
+
+func treatAsProxyDriver(labels map[string]string) error {
+	isProxyDriver := config.GetFsDriver() == config.FsDriverProxy
+	isProxyLabel := label.IsNydusProxyMode(labels)
+	_, isProxyImage := labels[label.CRIImageRef]
+	log.G(context.Background()).Debugf("isProxyDriver = %t, isProxyLabel = %t, isProxyImage = %t", isProxyDriver, isProxyLabel, isProxyImage)
+	switch {
+	case isProxyDriver && isProxyImage:
+		return nil
+	case isProxyDriver != isProxyLabel:
+		return errors.Errorf("check Labels With Driver failed, driver = %q, labels = %q", config.GetFsDriver(), labels)
+	default:
+		return nil
+	}
+}