feat: mount additional disks when using vz

Signed-off-by: Justin Alvarez <alvajus@amazon.com>

README.md

@@ -224,7 +224,7 @@ Use `<INSTANCE>:<FILENAME>` to specify a source or target inside an instance.
 
 #### `limactl disk`
 
-`limactl disk create <DISK> --size <SIZE>`: create a new external disk to attach to an instance
+`limactl disk create <DISK> --size <SIZE> [--format qcow2]`: create a new external disk to attach to an instance
 
 `limactl disk delete <DISK>`: delete an existing disk
 

cmd/limactl/disk.go

@@ -20,7 +20,7 @@ func newDiskCommand() *cobra.Command {
 		Use:   "disk",
 		Short: "Lima disk management",
 		Example: `  Create a disk:
-  $ limactl disk create DISK --size SIZE
+  $ limactl disk create DISK --size SIZE [--format qcow2]
 
   List existing disks:
   $ limactl disk ls
@@ -44,14 +44,15 @@ func newDiskCreateCommand() *cobra.Command {
 		Use: "create DISK",
 		Example: `
 To create a new disk:
-$ limactl disk create DISK --size SIZE
+$ limactl disk create DISK --size SIZE [--format qcow2]
 `,
 		Short: "Create a Lima disk",
 		Args:  WrapArgsError(cobra.ExactArgs(1)),
 		RunE:  diskCreateAction,
 	}
 	diskCreateCommand.Flags().String("size", "", "configure the disk size")
 	diskCreateCommand.MarkFlagRequired("size")
+	diskCreateCommand.Flags().String("format", "qcow2", "specify the disk format")
 	return diskCreateCommand
 }
 
@@ -61,11 +62,22 @@ func diskCreateAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	format, err := cmd.Flags().GetString("format")
+	if err != nil {
+		return err
+	}
+
 	diskSize, err := units.RAMInBytes(size)
 	if err != nil {
 		return err
 	}
 
+	switch format {
+	case "qcow2", "raw":
+	default:
+		return fmt.Errorf(`disk format %q not supported, use "qcow2" or "raw" instead`, format)
+	}
+
 	// only exactly one arg is allowed
 	name := args[0]
 
@@ -78,13 +90,13 @@ func diskCreateAction(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("disk %q already exists (%q)", name, diskDir)
 	}
 
-	logrus.Infof("Creating disk %q with size %s", name, units.BytesSize(float64(diskSize)))
+	logrus.Infof("Creating %s disk %q with size %s", format, name, units.BytesSize(float64(diskSize)))
 
 	if err := os.MkdirAll(diskDir, 0700); err != nil {
 		return err
 	}
 
-	if err := qemu.CreateDataDisk(diskDir, int(diskSize)); err != nil {
+	if err := qemu.CreateDataDisk(diskDir, format, int(diskSize)); err != nil {
 		return err
 	}
 

docs/internal.md

@@ -74,11 +74,19 @@ Host agent:
 A disk directory contains the following files:
 
 data disk:
-- `datadisk`: the qcow2 disk that is attached to an instance
+- `datadisk`: the qcow2 or raw disk that is attached to an instance
 
 lock:
 - `in_use_by`: symlink to the instance directory that is using the disk
 
+When using `vmType: vz` (Virtualization.framework), on boot, any qcow2 (default) formatted disks that are specified in `additionalDisks` will be converted to RAW since [Virtualization.framework only supports mounting RAW disks](https://developer.apple.com/documentation/virtualization/vzdiskimagestoragedeviceattachment). This conversion enables additional disks to work with both Virtualization.framework and QEMU, but it has some consequences when it comes to interacting with the disk, most importantly that a regular macOS default `cp` command, it will copy the _entire_ disk, instead of just the _used_ portion. There are a few workarounds for this:
+- `qemu-img convert -f raw -O raw old_path new_path` will do a sparse copy
+- GNU's `cp` command is sparse-aware and can be installed with `brew install coreutils` (by default, the new GNU utils are not added to `$PATH`)
+- `rsync --sparse old_path new_path` can be used in place of `cp` to do a sparse copy
+  - macOS ships with an older version of rsync that is apparently not sparse-aware. Using the version from `brew install rsync` will work (the brew version will be added to `$PATH`, but the macOS default version will come first)
+
+`du -h disk_path` or `qemu-img info disk_path` can be used to see the real size of RAW disks (`ls` will only show the full size). See [#1405](https://github.com/lima-vm/lima/pull/1405) for more details.
+
 ## Lima cache directory (`~/Library/Caches/lima`)
 
 Currently hard-coded to `~/Library/Caches/lima` on macOS.

pkg/driver/driver.go

@@ -12,6 +12,8 @@ type Driver interface {
 
 	CreateDisk() error
 
+	InspectDisk(diskName string) (*store.Disk, error)
+
 	Start(_ context.Context) (chan error, error)
 
 	Stop(_ context.Context) error
@@ -36,6 +38,14 @@ func (d *BaseDriver) CreateDisk() error {
 	return nil
 }
 
+func (d *BaseDriver) InspectDisk(diskName string) (*store.Disk, error) {
+	disk, err := store.InspectDisk(diskName)
+	if err != nil {
+		return nil, err
+	}
+	return disk, nil
+}
+
 func (d *BaseDriver) Start(_ context.Context) (chan error, error) {
 	return nil, nil
 }

pkg/qemu/qemu.go

@@ -104,14 +104,14 @@ func EnsureDisk(cfg Config) error {
 	return nil
 }
 
-func CreateDataDisk(dir string, size int) error {
+func CreateDataDisk(dir, format string, size int) error {
 	dataDisk := filepath.Join(dir, filenames.DataDisk)
 	if _, err := os.Stat(dataDisk); err == nil || !errors.Is(err, fs.ErrNotExist) {
 		// datadisk already exists
 		return err
 	}
 
-	args := []string{"create", "-f", "qcow2", dataDisk, strconv.Itoa(size)}
+	args := []string{"create", "-f", format, dataDisk, strconv.Itoa(size)}
 	cmd := exec.Command("qemu-img", args...)
 	if out, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("failed to run %v: %q: %w", cmd.Args, string(out), err)

pkg/vz/vm_darwin.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"syscall"
@@ -334,6 +335,55 @@ func attachDisks(driver *driver.BaseDriver, vmConfig *vz.VirtualMachineConfigura
 	}
 	configurations = append(configurations, diffDisk)
 
+	for _, diskName := range driver.Yaml.AdditionalDisks {
+		d, err := driver.InspectDisk(diskName)
+		if err != nil {
+			return fmt.Errorf("failed to run load disk %q: %q", diskName, err)
+		}
+
+		if d.Instance != "" {
+			return fmt.Errorf("failed to run attach disk %q, in use by instance %q", diskName, d.Instance)
+		}
+		logrus.Infof("Mounting disk %q on %q", diskName, d.MountPoint)
+		err = d.Lock(driver.Instance.Dir)
+		if err != nil {
+			return fmt.Errorf("failed to run lock disk %q: %q", diskName, err)
+		}
+		extraDiskPath := filepath.Join(d.Dir, filenames.DataDisk)
+
+		extraDiskFormat, err := imgutil.DetectFormat(extraDiskPath)
+		if err != nil {
+			return fmt.Errorf("failed to run detect disk format %q: %q", diskName, err)
+		}
+		if extraDiskFormat != "raw" {
+			rawPath := fmt.Sprintf("%s.raw", extraDiskPath)
+			if err = imgutil.QCOWToRaw(extraDiskPath, rawPath); err != nil {
+				return fmt.Errorf("failed to convert qcow2 disk %q to raw for vz driver: %w", diskName, err)
+			}
+			cmd := exec.Command("mv", extraDiskPath, fmt.Sprintf("%s.qcow2", extraDiskPath))
+			if out, err := cmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("failed to run %v: %q: %w", cmd.Args, string(out), err)
+			}
+			cmd = exec.Command("mv", rawPath, extraDiskPath)
+			if out, err := cmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("failed to run %v: %q: %w", cmd.Args, string(out), err)
+			}
+		}
+
+		if err = validateDiskFormat(extraDiskPath); err != nil {
+			return fmt.Errorf("failed to validate extra disk %q: %w", extraDiskPath, err)
+		}
+		extraDiskPathAttachment, err := vz.NewDiskImageStorageDeviceAttachmentWithCacheAndSync(extraDiskPath, false, vz.DiskImageCachingModeAutomatic, vz.DiskImageSynchronizationModeFsync)
+		if err != nil {
+			return fmt.Errorf("failed to create disk attachment for extra disk %q: %w", extraDiskPath, err)
+		}
+		extraDisk, err := vz.NewVirtioBlockDeviceConfiguration(extraDiskPathAttachment)
+		if err != nil {
+			return fmt.Errorf("failed to create new virtio block device config for extra disk %q: %w", extraDiskPath, err)
+		}
+		configurations = append(configurations, extraDisk)
+	}
+
 	if err = validateDiskFormat(ciDataPath); err != nil {
 		return err
 	}

pkg/vz/vz_driver_darwin.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/lima-vm/lima/pkg/reflectutil"
+	"github.com/lima-vm/lima/pkg/store"
 
 	"github.com/Code-Hex/vz/v3"
 
@@ -64,6 +65,7 @@ func (l *LimaVzDriver) Validate() error {
 		"PropagateProxyEnv",
 		"CACertificates",
 		"Rosetta",
+		"AdditionalDisks",
 	); len(unknown) > 0 {
 		logrus.Warnf("Ignoring: vmType %s: %+v", *l.Yaml.VMType, unknown)
 	}
@@ -106,6 +108,10 @@ func (l *LimaVzDriver) CreateDisk() error {
 	return nil
 }
 
+func (l *LimaVzDriver) InspectDisk(diskName string) (*store.Disk, error) {
+	return l.BaseDriver.InspectDisk(diskName)
+}
+
 func (l *LimaVzDriver) Start(ctx context.Context) (chan error, error) {
 	logrus.Infof("Starting VZ (hint: to watch the boot progress, see %q)", filepath.Join(l.Instance.Dir, filenames.SerialLog))
 	vm, errCh, err := startVM(ctx, l.BaseDriver)