Implement Windows mounting for bind and windows-layer mounts

Using symlinks for bind mounts means we are not protecting an RO-mounted
layer against modification. Windows doesn't currently appear to offer a
better approach though, as we cannot create arbitrary empty WCOW scratch
layers at this time.

For windows-layer mounts, Unmount does not have access to the mounts
used to create it. So we store the relevant data in an Alternate Data
Stream on the mountpoint in order to be able to Unmount later.

Based on approach in #2366,
with sign-offs recorded as 'Based-on-work-by' trailers below.

This also partially-reverts some changes made in #6034 as they are not
needed with this mounting implmentation, which no longer needs to be
handled specially by the caller compared to non-Windows mounts.

Signed-off-by: Paul "TBBle" Hampson <Paul.Hampson@Pobox.com>
Based-on-work-by: Michael Crosby <crosbymichael@gmail.com>
Based-on-work-by: Darren Stahl <darst@microsoft.com>

mount/mount_windows.go

@@ -18,6 +18,7 @@ package mount
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -26,8 +27,22 @@ import (
 	"github.com/Microsoft/hcsshim"
 )
 
+const sourceStreamName = "containerd.io-source"
+
+var (
+	// ErrNotImplementOnWindows is returned when an action is not implemented for windows
+	ErrNotImplementOnWindows = errors.New("not implemented under windows")
+)
+
 // Mount to the provided target.
 func (m *Mount) mount(target string) error {
+	if m.Type == "bind" {
+		if err := m.bindMount(target); err != nil {
+			return fmt.Errorf("failed to bind-mount to %s: %w", target, err)
+		}
+		return nil
+	}
+
 	if m.Type != "windows-layer" {
 		return fmt.Errorf("invalid windows mount type: '%s'", m.Type)
 	}
@@ -46,22 +61,42 @@ func (m *Mount) mount(target string) error {
 	if err = hcsshim.ActivateLayer(di, layerID); err != nil {
 		return fmt.Errorf("failed to activate layer %s: %w", m.Source, err)
 	}
+	defer func() {
+		if err != nil {
+			hcsshim.DeactivateLayer(di, layerID)
+		}
+	}()
 
 	if err = hcsshim.PrepareLayer(di, layerID, parentLayerPaths); err != nil {
 		return fmt.Errorf("failed to prepare layer %s: %w", m.Source, err)
 	}
+	defer func() {
+		if err != nil {
+			hcsshim.UnprepareLayer(di, layerID)
+		}
+	}()
 
-	// We can link the layer mount path to the given target. It is an UNC path, and it needs
-	// a trailing backslash.
-	mountPath, err := hcsshim.GetLayerMountPath(di, layerID)
+	volume, err := hcsshim.GetLayerMountPath(di, layerID)
 	if err != nil {
-		return fmt.Errorf("failed to get layer mount path for %s: %w", m.Source, err)
+		return fmt.Errorf("failed to get volume path for layer %s: %w", m.Source, err)
+	}
+
+	if err = setVolumeMountPoint(target, volume); err != nil {
+		return fmt.Errorf("failed to set volume mount path for layer %s: %w", m.Source, err)
 	}
-	mountPath = mountPath + `\`
+	defer func() {
+		if err != nil {
+			deleteVolumeMountPoint(target)
+		}
+	}()
 
-	if err = os.Symlink(mountPath, target); err != nil {
-		return fmt.Errorf("failed to link mount to target %s: %w", target, err)
+	// Add an Alternate Data Stream to record the layer source.
+	// See https://docs.microsoft.com/en-au/archive/blogs/askcore/alternate-data-streams-in-ntfs
+	// for details on Alternate Data Streams.
+	if err = os.WriteFile(filepath.Clean(target)+":"+sourceStreamName, []byte(m.Source), 0666); err != nil {
+		return fmt.Errorf("failed to record source for layer %s: %w", m.Source, err)
 	}
+
 	return nil
 }
 
@@ -85,8 +120,37 @@ func (m *Mount) GetParentPaths() ([]string, error) {
 
 // Unmount the mount at the provided path
 func Unmount(mount string, flags int) error {
+	mount = filepath.Clean(mount)
+
+	// Helpfully, both reparse points and symlinks look like links to Go
+	// Less-helpfully, ReadLink cannot return \\?\Volume{GUID} for a volume mount,
+	// and ends up returning the directory we gave it for some reason.
+	if mountTarget, err := os.Readlink(mount); err != nil {
+		// Not a mount point.
+		// This isn't an error, per the EINVAL handling in the Linux version
+		return nil
+	} else if mount != filepath.Clean(mountTarget) {
+		// Directory symlink
+		if err := bindUnmount(mount); err != nil {
+			return fmt.Errorf("failed to bind-unmount from %s: %w", mount, err)
+		}
+		return nil
+	}
+
+	layerPathb, err := os.ReadFile(mount + ":" + sourceStreamName)
+
+	if err != nil {
+		return fmt.Errorf("failed to retrieve source for layer %s: %w", mount, err)
+	}
+
+	layerPath := string(layerPathb)
+
+	if err := deleteVolumeMountPoint(mount); err != nil {
+		return fmt.Errorf("failed failed to release volume mount path for layer %s: %w", mount, err)
+	}
+
 	var (
-		home, layerID = filepath.Split(mount)
+		home, layerID = filepath.Split(layerPath)
 		di            = hcsshim.DriverInfo{
 			HomeDir: home,
 		}
@@ -95,6 +159,7 @@ func Unmount(mount string, flags int) error {
 	if err := hcsshim.UnprepareLayer(di, layerID); err != nil {
 		return fmt.Errorf("failed to unprepare layer %s: %w", mount, err)
 	}
+
 	if err := hcsshim.DeactivateLayer(di, layerID); err != nil {
 		return fmt.Errorf("failed to deactivate layer %s: %w", mount, err)
 	}
@@ -104,10 +169,35 @@ func Unmount(mount string, flags int) error {
 
 // UnmountAll unmounts from the provided path
 func UnmountAll(mount string, flags int) error {
+	if mount == "" {
+		// This isn't an error, per the EINVAL handling in the Linux version
+		return nil
+	}
+
 	return Unmount(mount, flags)
 }
 
 // UnmountRecursive unmounts from the provided path
 func UnmountRecursive(mount string, flags int) error {
 	return UnmountAll(mount, flags)
 }
+
+func (m *Mount) bindMount(target string) error {
+	for _, option := range m.Options {
+		if option == "ro" {
+			return fmt.Errorf("read-only bind mount: %w", ErrNotImplementOnWindows)
+		}
+	}
+
+	if err := os.Remove(target); err != nil {
+		return err
+	}
+
+	// TODO: We don't honour the Read-Only flag.
+	// It's possible that Windows simply lacks this.
+	return os.Symlink(m.Source, target)
+}
+
+func bindUnmount(target string) error {
+	return os.Remove(target)
+}

mount/volumemountutils_windows.go

@@ -0,0 +1,75 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package mount
+
+// Simple wrappers around SetVolumeMountPoint and DeleteVolumeMountPoint
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/containerd/containerd/errdefs"
+	"golang.org/x/sys/windows"
+)
+
+// Mount volumePath (in format '\\?\Volume{GUID}' at targetPath.
+// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setvolumemountpointw
+func setVolumeMountPoint(targetPath string, volumePath string) error {
+	if !strings.HasPrefix(volumePath, "\\\\?\\Volume{") {
+		return fmt.Errorf("unable to mount non-volume path %s: %w", volumePath, errdefs.ErrInvalidArgument)
+	}
+
+	// Both must end in a backslash
+	slashedTarget := filepath.Clean(targetPath) + string(filepath.Separator)
+	slashedVolume := volumePath + string(filepath.Separator)
+
+	targetP, err := syscall.UTF16PtrFromString(slashedTarget)
+	if err != nil {
+		return fmt.Errorf("unable to utf16-ise %s: %w", slashedTarget, err)
+	}
+
+	volumeP, err := syscall.UTF16PtrFromString(slashedVolume)
+	if err != nil {
+		return fmt.Errorf("unable to utf16-ise %s: %w", slashedVolume, err)
+	}
+
+	if err := windows.SetVolumeMountPoint(targetP, volumeP); err != nil {
+		return fmt.Errorf("failed calling SetVolumeMount('%s', '%s'): %w", slashedTarget, slashedVolume, err)
+	}
+
+	return nil
+}
+
+// Remove the volume mount at targetPath
+// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-deletevolumemountpointa
+func deleteVolumeMountPoint(targetPath string) error {
+	// Must end in a backslash
+	slashedTarget := filepath.Clean(targetPath) + string(filepath.Separator)
+
+	targetP, err := syscall.UTF16PtrFromString(slashedTarget)
+	if err != nil {
+		return fmt.Errorf("unable to utf16-ise %s: %w", slashedTarget, err)
+	}
+
+	if err := windows.DeleteVolumeMountPoint(targetP); err != nil {
+		return fmt.Errorf("failed calling DeleteVolumeMountPoint('%s'): %w", slashedTarget, err)
+	}
+
+	return nil
+}

pkg/cri/opts/container.go

@@ -21,8 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
-	goruntime "runtime"
 	"strings"
 
 	"github.com/containerd/continuity/fs"
@@ -86,53 +84,34 @@ func WithVolumes(volumeMounts map[string]string) containerd.NewContainerOpts {
 		// https://github.com/containerd/containerd/pull/1785
 		defer os.Remove(root)
 
-		unmounter := func(mountPath string) {
-			if uerr := mount.Unmount(mountPath, 0); uerr != nil {
+		if err := mount.All(mounts, root); err != nil {
+			return fmt.Errorf("failed to mount: %w", err)
+		}
+		defer func() {
+			if uerr := mount.Unmount(root, 0); uerr != nil {
 				log.G(ctx).WithError(uerr).Errorf("Failed to unmount snapshot %q", root)
 				if err == nil {
 					err = uerr
 				}
 			}
-		}
-
-		var mountPaths []string
-		if goruntime.GOOS == "windows" {
-			for _, m := range mounts {
-				// appending the layerID to the root.
-				mountPath := filepath.Join(root, filepath.Base(m.Source))
-				mountPaths = append(mountPaths, mountPath)
-				if err := m.Mount(mountPath); err != nil {
-					return err
-				}
-
-				defer unmounter(m.Source)
-			}
-		} else {
-			mountPaths = append(mountPaths, root)
-			if err := mount.All(mounts, root); err != nil {
-				return fmt.Errorf("failed to mount: %w", err)
-			}
-			defer unmounter(root)
-		}
+		}()
 
 		for host, volume := range volumeMounts {
 			// The volume may have been defined with a C: prefix, which we can't use here.
 			volume = strings.TrimPrefix(volume, "C:")
-			for _, mountPath := range mountPaths {
-				src, err := fs.RootPath(mountPath, volume)
-				if err != nil {
-					return fmt.Errorf("rootpath on mountPath %s, volume %s: %w", mountPath, volume, err)
-				}
-				if _, err := os.Stat(src); err != nil {
-					if os.IsNotExist(err) {
-						// Skip copying directory if it does not exist.
-						continue
-					}
-					return fmt.Errorf("stat volume in rootfs: %w", err)
-				}
-				if err := copyExistingContents(src, host); err != nil {
-					return fmt.Errorf("taking runtime copy of volume: %w", err)
+			src, err := fs.RootPath(root, volume)
+			if err != nil {
+				return fmt.Errorf("rootpath on mountPath %s, volume %s: %w", root, volume, err)
+			}
+			if _, err := os.Stat(src); err != nil {
+				if os.IsNotExist(err) {
+					// Skip copying directory if it does not exist.
+					continue
 				}
+				return fmt.Errorf("stat volume in rootfs: %w", err)
+			}
+			if err := copyExistingContents(src, host); err != nil {
+				return fmt.Errorf("taking runtime copy of volume: %w", err)
 			}
 		}
 		return nil