Add QEMU microvm hypervisor backend

DEVELOPMENT.md

@@ -16,7 +16,7 @@ See [macOS Development](#macos-development) below for native macOS development u
 
 **Linux Prerequisites:**
 
-**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq**
+**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq**. QEMU development and testing also require the architecture-appropriate `qemu-system` package (for example, `qemu-system-x86` on amd64).
 
 ```bash
 # Verify prerequisites
@@ -27,7 +27,7 @@ dnsmasq --version
 **Install on Debian/Ubuntu:**
 
 ```bash
-sudo apt-get install erofs-utils dnsmasq
+sudo apt-get install erofs-utils dnsmasq qemu-system-x86
 ```
 
 **KVM Access:** User must be in `kvm` group for VM access:
@@ -126,11 +126,15 @@ Common settings:
 | `acme.dns_provider` | DNS provider for ACME challenges | _(empty)_ |
 | `acme.cloudflare_api_token` | Cloudflare API token | _(empty)_ |
 | `build.docker_socket` | Path to Docker socket | `/var/run/docker.sock` |
-| `hypervisor.default` | Default hypervisor type (`cloud-hypervisor`, `firecracker`, `qemu`, `vz`) | `cloud-hypervisor` |
+| `hypervisor.default` | Default hypervisor type (`cloud-hypervisor`, `firecracker`, `qemu`, `qemu-microvm`, `vz`) | `cloud-hypervisor` |
 | `hypervisor.firecracker_binary_path` | Optional runtime path to external Firecracker binary | _(empty = embedded)_ |
 
 Environment variables can also override any config key using `__` as the nesting separator (e.g. `CADDY__LISTEN_ADDRESS` overrides `caddy.listen_address`).
 
+### QEMU backends
+
+`qemu` uses QEMU's architecture-native standard board (`q35` on amd64 and `virt` on arm64). `qemu-microvm` is a Linux amd64-only backend using QEMU's minimal `microvm` board and virtio-mmio devices. It does not support PCI/vGPU passthrough, hotplug memory, or more than eight virtio-mmio devices. Standby restore and warm forks require the exact QEMU version recorded when the memory image was written. After an upgrade, restore a stopped snapshot or recreate the instance; instances already in `Stopped` state still cold-start normally.
+
 **Important: Subnet Configuration**
 
 The default subnet `10.100.0.0/16` is chosen to avoid common conflicts. Hypeman will detect conflicts with existing routes on startup and fail with guidance.

Makefile

@@ -301,6 +301,8 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries
 			"HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \
 			"HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \
 			"HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \
+			"HYPEMAN_QEMU_BOOT_BENCH=$${HYPEMAN_QEMU_BOOT_BENCH:-}" \
+			"HYPEMAN_QEMU_BOOT_BENCH_SAMPLES=$${HYPEMAN_QEMU_BOOT_BENCH_SAMPLES:-}" \
 			go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
 	else \
 		sudo env "PATH=$$TEST_PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" "CI=$${CI:-}" \
@@ -310,6 +312,8 @@ test-linux: ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries
 			"HYPEMAN_TEST_PREWARM_DIR=$${HYPEMAN_TEST_PREWARM_DIR:-}" \
 			"HYPEMAN_TEST_PREWARM_STRICT=$${HYPEMAN_TEST_PREWARM_STRICT:-}" \
 			"HYPEMAN_TEST_REGISTRY=$${HYPEMAN_TEST_REGISTRY:-}" \
+			"HYPEMAN_QEMU_BOOT_BENCH=$${HYPEMAN_QEMU_BOOT_BENCH:-}" \
+			"HYPEMAN_QEMU_BOOT_BENCH_SAMPLES=$${HYPEMAN_QEMU_BOOT_BENCH_SAMPLES:-}" \
 			go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=$(TEST_TIMEOUT) ./...; \
 	fi
 

README.md

@@ -112,6 +112,10 @@ hypeman pull nginx:alpine
 # Boot a new VM (auto-pulls image if needed)
 hypeman run --name my-app nginx:alpine
 
+# On Linux amd64, use QEMU's minimal microvm backend.
+# It cannot use PCI devices or hotplug memory.
+hypeman run --hypervisor qemu-microvm --name my-microvm nginx:alpine
+
 # List running VMs
 hypeman ps
 

cmd/api/api/instances.go

@@ -247,7 +247,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	if request.Body.Hypervisor != nil {
 		hvType = hypervisor.Type(*request.Body.Hypervisor)
 	}
-
 	// Parse GPU configuration (vGPU mode)
 	var gpuConfig *instances.GPUConfig
 	if request.Body.Gpu != nil && request.Body.Gpu.Profile != nil && *request.Body.Gpu.Profile != "" {
@@ -1149,7 +1148,6 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		HasSnapshot: lo.ToPtr(inst.HasSnapshot),
 		Hypervisor:  &hvType,
 	}
-
 	if b, err := json.Marshal(networkPayload); err == nil {
 		_ = json.Unmarshal(b, &oapiInst.Network)
 	}

cmd/api/api/instances_test.go

@@ -316,6 +316,37 @@ func (m *captureCreateManager) CreateInstance(ctx context.Context, req instances
 	}, nil
 }
 
+func TestCreateInstance_MapsQEMUMicroVMHypervisor(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	mockMgr := &captureCreateManager{Manager: svc.InstanceManager}
+	svc.InstanceManager = mockMgr
+	microvm := oapi.CreateInstanceRequestHypervisor(hypervisor.TypeQEMUMicroVM)
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{
+		Name:       "test-qemu-microvm",
+		Image:      "docker.io/library/alpine:latest",
+		Hypervisor: &microvm,
+	}})
+	require.NoError(t, err)
+	require.IsType(t, oapi.CreateInstance201JSONResponse{}, resp)
+	require.NotNil(t, mockMgr.lastReq)
+	assert.Equal(t, hypervisor.TypeQEMUMicroVM, mockMgr.lastReq.Hypervisor)
+}
+
+func TestInstanceToOAPI_QEMUMicroVMHypervisor(t *testing.T) {
+	t.Parallel()
+
+	inst := instances.Instance{StoredMetadata: instances.StoredMetadata{
+		Id: "qemu-microvm", Name: "qemu-microvm", Image: "alpine", CreatedAt: time.Now(),
+		HypervisorType: hypervisor.TypeQEMUMicroVM,
+	}}
+	oapiInst := instanceToOAPI(inst)
+	require.NotNil(t, oapiInst.Hypervisor)
+	assert.Equal(t, string(hypervisor.TypeQEMUMicroVM), string(*oapiInst.Hypervisor))
+}
+
 func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) {
 	t.Parallel()
 	svc := newTestService(t)

lib/hypervisor/README.md

@@ -13,12 +13,12 @@ Hypeman originally supported only Cloud Hypervisor. This abstraction layer allow
 
 ## Implementations
 
-| Hypervisor | Platform | Process Model | Control Interface |
-|------------|----------|---------------|-------------------|
-| Cloud Hypervisor | Linux | External process | HTTP API over Unix socket |
-| Firecracker | Linux | External process | HTTP API over Unix socket |
-| QEMU | Linux | External process | QMP over Unix socket |
-| vz | macOS | Subprocess (vz-shim) | HTTP API over Unix socket |
+| Hypervisor | Platform | Process Model | Control Interface | Constraints |
+|------------|----------|---------------|-------------------|-------------|
+| Cloud Hypervisor | Linux | External process | HTTP API over Unix socket | — |
+| Firecracker | Linux | External process | HTTP API over Unix socket | — |
+| QEMU | Linux | External process | QMP over Unix socket | `qemu` uses q35/virt; `qemu-microvm` uses the amd64 microvm board without PCI/hotplug support |
+| vz | macOS | Subprocess (vz-shim) | HTTP API over Unix socket | — |
 
 ## How It Works
 

lib/hypervisor/config.go

@@ -1,8 +1,14 @@
 package hypervisor
 
+// MachineType identifies a hypervisor-specific machine/board type.
+type MachineType string
+
 // VMConfig is the hypervisor-agnostic VM configuration.
 // Each hypervisor implementation translates this to its native format.
 type VMConfig struct {
+	// MachineType selects a hypervisor-specific board. It is currently used by QEMU.
+	MachineType MachineType
+
 	// Compute resources
 	VCPUs        int
 	MemoryBytes  int64

lib/hypervisor/hypervisor.go

@@ -31,8 +31,10 @@ const (
 	TypeCloudHypervisor Type = "cloud-hypervisor"
 	// TypeFirecracker is the Firecracker VMM
 	TypeFirecracker Type = "firecracker"
-	// TypeQEMU is the QEMU VMM
+	// TypeQEMU is QEMU with its architecture-native standard board.
 	TypeQEMU Type = "qemu"
+	// TypeQEMUMicroVM is QEMU with the minimal x86 microvm board.
+	TypeQEMUMicroVM Type = "qemu-microvm"
 	// TypeVZ is the Virtualization.framework VMM (macOS only)
 	TypeVZ Type = "vz"
 )
@@ -252,6 +254,11 @@ type Capabilities struct {
 	// on-disk base across restore/standby cycles.
 	SupportsSnapshotBaseReuse bool
 
+	// RequiresExactSnapshotVersion indicates memory snapshots can only restore
+	// with the exact hypervisor version that wrote them. Generic lifecycle code
+	// uses this to keep persisted version metadata strict and current.
+	RequiresExactSnapshotVersion bool
+
 	// SupportsConcurrentForkPrepare indicates stopped/standby forks can prepare
 	// separate target snapshots concurrently from the same source.
 	SupportsConcurrentForkPrepare bool

lib/hypervisor/qemu/README.md

@@ -0,0 +1,15 @@
+# QEMU hypervisor
+
+The `qemu` backend uses `q35` on amd64 and `virt` on arm64. These architecture-native standard boards are selected internally and are not public API options.
+
+## `qemu-microvm`
+
+The `qemu-microvm` backend uses QEMU's Linux amd64-only `microvm` board. Hypeman uses direct kernel boot, `ttyS0` serial logs, and virtio-mmio transport for disks, networking, vsock, and the optional balloon.
+
+It cannot use PCI/VFIO/vGPU devices or hotplug memory. QEMU limits it to eight virtio-mmio devices; Hypeman counts rootfs, overlay, config disk, attached-volume disks (an overlay volume consumes two), network, vsock, and the optional balloon before starting QEMU.
+
+A `qemu-microvm` standby snapshot or warm fork may restore only with the exact QEMU version that wrote its memory image. Hypeman records the running binary's version in `qemu-config.json` and checks it before restore. If QEMU changes, restore a stopped snapshot with `target_state: Stopped` and start it normally, or recreate the instance; an instance already in `Stopped` state can always cold-start. A stopped snapshot may switch between `qemu`, `qemu-microvm`, and other hypervisors; the target backend determines the internal QEMU board.
+
+## Boot comparison
+
+`./scripts/benchmark-qemu-machine-types.sh [samples]` is an opt-in, non-gating Linux/KVM benchmark. It reports p50/p95 `StartedAt` → `ProgramStartedAt` latency and QEMU RSS for equivalent q35 and microvm nginx guests.

lib/hypervisor/qemu/config.go

@@ -3,7 +3,6 @@ package qemu
 import (
 	"fmt"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 
@@ -13,9 +12,15 @@ import (
 // BuildArgs converts hypervisor.VMConfig to QEMU command-line arguments.
 func BuildArgs(cfg hypervisor.VMConfig) []string {
 	args := make([]string, 0, 64)
-
-	// Machine type with KVM acceleration (arch-specific)
-	args = append(args, "-machine", machineType())
+	microvm := cfg.MachineType == MachineTypeMicroVM
+
+	// Machine type with KVM acceleration (arch-specific when omitted).
+	args = append(args, "-machine", machineTypeForConfig(cfg))
+	if microvm {
+		// Do not allow a host qemu.conf to add devices outside microvm's
+		// documented eight virtio-mmio-device limit.
+		args = append(args, "-no-user-config")
+	}
 
 	// CPU configuration
 	args = append(args, "-cpu", "host")
@@ -26,7 +31,7 @@ func BuildArgs(cfg hypervisor.VMConfig) []string {
 	args = append(args, "-m", fmt.Sprintf("%dM", memMB))
 
 	if cfg.GuestMemory.EnableBalloon {
-		balloonOpts := []string{"virtio-balloon-pci"}
+		balloonOpts := []string{virtioDevice(microvm, "virtio-balloon")}
 		if cfg.GuestMemory.DeflateOnOOM {
 			balloonOpts = append(balloonOpts, "deflate-on-oom=on")
 		}
@@ -65,21 +70,21 @@ func BuildArgs(cfg hypervisor.VMConfig) []string {
 			}
 		}
 		args = append(args, "-drive", driveOpts)
-		args = append(args, "-device", fmt.Sprintf("virtio-blk-pci,drive=drive%d", i))
+		args = append(args, "-device", fmt.Sprintf("%s,drive=drive%d", virtioDevice(microvm, "virtio-blk"), i))
 	}
 
 	// Network configuration
 	for i, net := range cfg.Networks {
 		netdevOpts := fmt.Sprintf("tap,id=net%d,ifname=%s,script=no,downscript=no", i, net.TAPDevice)
 		args = append(args, "-netdev", netdevOpts)
 
-		deviceOpts := fmt.Sprintf("virtio-net-pci,netdev=net%d,mac=%s", i, net.MAC)
+		deviceOpts := fmt.Sprintf("%s,netdev=net%d,mac=%s", virtioDevice(microvm, "virtio-net"), i, net.MAC)
 		args = append(args, "-device", deviceOpts)
 	}
 
 	// Vsock configuration
 	if cfg.VsockCID > 0 {
-		args = append(args, "-device", fmt.Sprintf("vhost-vsock-pci,guest-cid=%d", cfg.VsockCID))
+		args = append(args, "-device", fmt.Sprintf("%s,guest-cid=%d", virtioDevice(microvm, "vhost-vsock"), cfg.VsockCID))
 	}
 
 	// PCI device passthrough (GPU, mdev vGPU, etc.)
@@ -123,13 +128,19 @@ func BuildArgs(cfg hypervisor.VMConfig) []string {
 	return args
 }
 
-// machineType returns the QEMU machine type for the host architecture.
-func machineType() string {
-	switch runtime.GOARCH {
-	case "arm64":
-		return "virt,accel=kvm"
-	default:
-		// x86_64 and others use q35
-		return "q35,accel=kvm"
+func machineTypeForConfig(cfg hypervisor.VMConfig) string {
+	machine := cfg.MachineType
+	if machine == "" {
+		// StartVM resolves the architecture default before calling BuildArgs. Keep
+		// direct BuildArgs callers deterministic for the standard QEMU profile.
+		machine, _ = ResolveMachineType("")
+	}
+	return string(machine) + ",accel=kvm"
+}
+
+func virtioDevice(microvm bool, name string) string {
+	if microvm {
+		return name + "-device"
 	}
+	return name + "-pci"
 }

lib/hypervisor/qemu/config_test.go

@@ -20,7 +20,7 @@ func TestBuildArgs_Basic(t *testing.T) {
 
 	// Check machine type (arch-dependent)
 	assert.Contains(t, args, "-machine")
-	assert.Contains(t, args, machineType())
+	assert.Contains(t, args, machineTypeForConfig(cfg))
 
 	// Check CPU
 	assert.Contains(t, args, "-cpu")
@@ -162,6 +162,31 @@ func TestBuildArgs_NoSerialLog(t *testing.T) {
 	assert.Contains(t, args, "stdio")
 }
 
+func TestBuildArgs_MicroVM(t *testing.T) {
+	cfg := hypervisor.VMConfig{
+		MachineType:   MachineTypeMicroVM,
+		VCPUs:         1,
+		MemoryBytes:   512 * 1024 * 1024,
+		Disks:         []hypervisor.DiskConfig{{Path: "/rootfs"}},
+		Networks:      []hypervisor.NetworkConfig{{TAPDevice: "tap0", MAC: "02:00:00:ab:cd:ef"}},
+		VsockCID:      123,
+		SerialLogPath: "/var/log/app.log",
+		GuestMemory:   hypervisor.GuestMemoryConfig{EnableBalloon: true},
+	}
+
+	args := BuildArgs(cfg)
+	assert.Contains(t, args, "microvm,accel=kvm")
+	assert.Contains(t, args, "-no-user-config")
+	assert.Contains(t, args, "virtio-blk-device,drive=drive0")
+	assert.Contains(t, args, "virtio-net-device,netdev=net0,mac=02:00:00:ab:cd:ef")
+	assert.Contains(t, args, "vhost-vsock-device,guest-cid=123")
+	assert.Contains(t, args, "virtio-balloon-device")
+	assert.Contains(t, args, "chardev:serial0")
+	for _, arg := range args {
+		assert.NotContains(t, arg, "-pci", "microvm cannot use PCI transport")
+	}
+}
+
 func TestBuildArgs_GuestMemoryBalloon(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/hypervisor/qemu/fork.go

@@ -19,7 +19,7 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq
 	}
 
 	snapshotDir := filepath.Dir(req.SnapshotConfigPath)
-	cfg, err := loadVMConfig(snapshotDir)
+	saved, err := loadVMConfig(snapshotDir)
 	if err != nil {
 		// The generic path points to CH's config.json; for QEMU, require qemu-config.json.
 		expectedPath := filepath.Join(snapshotDir, vmConfigFile)
@@ -28,6 +28,10 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq
 		}
 		return hypervisor.ForkPrepareResult{}, fmt.Errorf("load qemu snapshot config: %w", err)
 	}
+	cfg, err := s.applyMachineType(saved.VMConfig, true)
+	if err != nil {
+		return hypervisor.ForkPrepareResult{}, fmt.Errorf("select qemu fork machine type: %w", err)
+	}
 
 	if req.SourceDataDir != "" && req.TargetDataDir != "" && req.SourceDataDir != req.TargetDataDir {
 		cfg = rewriteQEMUConfigPaths(cfg, req.SourceDataDir, req.TargetDataDir)
@@ -60,7 +64,8 @@ func (s *Starter) PrepareFork(ctx context.Context, req hypervisor.ForkPrepareReq
 		}
 	}
 
-	if err := saveVMConfig(snapshotDir, cfg); err != nil {
+	saved.VMConfig = cfg
+	if err := saveVMConfig(snapshotDir, saved); err != nil {
 		return hypervisor.ForkPrepareResult{}, fmt.Errorf("write qemu snapshot config: %w", err)
 	}
 	return hypervisor.ForkPrepareResult{

lib/hypervisor/qemu/fork_test.go

@@ -18,12 +18,16 @@ func TestPrepareFork_NoSnapshotPathIsNoOp(t *testing.T) {
 }
 
 func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) {
-	starter := NewStarter()
+	if _, err := ResolveMachineType(MachineTypeMicroVM); err != nil {
+		t.Skipf("microvm is unavailable on this platform: %v", err)
+	}
+	starter := NewMicroVMStarter()
 	snapshotDir := t.TempDir()
 
 	sourceDir := "/src/guest"
 	targetDir := "/dst/guest"
 	initial := hypervisor.VMConfig{
+		MachineType:   MachineTypeMicroVM,
 		VCPUs:         2,
 		MemoryBytes:   2 * 1024 * 1024 * 1024,
 		SerialLogPath: sourceDir + "/logs/app.log",
@@ -45,7 +49,7 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) {
 			},
 		},
 	}
-	require.NoError(t, saveVMConfig(snapshotDir, initial))
+	require.NoError(t, saveVMConfig(snapshotDir, savedVMConfig{VMConfig: initial, QEMUVersion: "8.2.0"}))
 
 	result, err := starter.PrepareFork(context.Background(), hypervisor.ForkPrepareRequest{
 		SnapshotConfigPath: filepath.Join(snapshotDir, "config.json"),
@@ -67,6 +71,8 @@ func TestPrepareFork_RewritesSnapshotConfig(t *testing.T) {
 	updated, err := loadVMConfig(snapshotDir)
 	require.NoError(t, err)
 
+	assert.Equal(t, MachineTypeMicroVM, updated.MachineType)
+	assert.Equal(t, "8.2.0", updated.QEMUVersion)
 	assert.Equal(t, int64(54321), updated.VsockCID)
 	assert.Equal(t, targetDir+"/vsock/fork-vsock.sock", updated.VsockSocket)
 	assert.Equal(t, targetDir+"/logs/fork-app.log", updated.SerialLogPath)