Abstract vGPU devices behind a framework dispatch

lib/devices/GPU.md

@@ -235,6 +235,21 @@ To upgrade the NVIDIA driver version:
    - Run GPU passthrough E2E tests
    - Verify with real CUDA workloads (e.g., ollama inference)
 
+## Rolling Back vGPU Changes
+
+Before downgrading Hypeman or the host to a version that does not support the active vGPU framework:
+
+1. Stop or delete all vGPU instances while the current Hypeman version can release their assignments.
+2. Confirm `/resources` reports `used_slots: 0`.
+3. Confirm no assignments remain in either framework:
+   ```bash
+   test -z "$(find /sys/bus/mdev/devices -mindepth 1 -maxdepth 1 2>/dev/null)"
+   find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' -exec grep -H -v '^0$' {} +
+   ```
+4. Downgrade only after both checks are clean.
+
+If assignment cleanup fails, Hypeman retains the instance metadata so a compatible version can retry it. Do not remove that metadata manually while the assignment remains active.
+
 ## Troubleshooting
 
 ### No GPU shown in /resources

lib/devices/mdev_darwin.go

@@ -30,6 +30,10 @@ func ListMdevDevices() ([]MdevDevice, error) {
 	return []MdevDevice{}, nil
 }
 
+func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) {
+	return nil, ErrVGPUNotSupportedOnMacOS
+}
+
 // CreateMdev returns an error on macOS as mdev is not supported.
 func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevice, error) {
 	return nil, ErrVGPUNotSupportedOnMacOS
@@ -45,6 +49,13 @@ func IsMdevInUse(mdevUUID string) bool {
 	return false
 }
 
+func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error {
+	if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev {
+		return fmt.Errorf("unknown vGPU framework %q", framework)
+	}
+	return nil
+}
+
 // ReconcileMdevs is a no-op on macOS.
 func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error {
 	return nil

lib/devices/mdev_linux.go

@@ -126,7 +126,7 @@ func DiscoverVFs() ([]VirtualFunction, error) {
 		vfs = append(vfs, VirtualFunction{
 			PCIAddress: vfAddr,
 			ParentGPU:  parentGPU,
-			HasMdev:    hasMdev,
+			Allocated:  hasMdev,
 		})
 	}
 
@@ -253,7 +253,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof
 	// Group free VFs by parent GPU (done once, shared by all goroutines)
 	freeVFsByParent := make(map[string][]VirtualFunction)
 	for _, vf := range vfs {
-		if vf.HasMdev {
+		if vf.Allocated {
 			continue
 		}
 		freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf)
@@ -453,7 +453,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType
 	allGPUs := make(map[string]bool)
 	for _, vf := range vfs {
 		allGPUs[vf.ParentGPU] = true
-		if !vf.HasMdev {
+		if !vf.Allocated {
 			freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf)
 		}
 	}

lib/devices/types.go

@@ -60,7 +60,12 @@ func ValidateDeviceName(name string) bool {
 // GPUMode represents the host's GPU configuration mode
 type GPUMode string
 
+type VGPUFramework string
+
 const (
+	VGPUFrameworkNone VGPUFramework = ""
+	VGPUFrameworkMdev VGPUFramework = "mdev"
+
 	// GPUModePassthrough indicates whole GPU VFIO passthrough
 	GPUModePassthrough GPUMode = "passthrough"
 	// GPUModeVGPU indicates SR-IOV + mdev based vGPU
@@ -73,7 +78,16 @@ const (
 type VirtualFunction struct {
 	PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4"
 	ParentGPU  string `json:"parent_gpu"`  // e.g., "0000:82:00.0"
-	HasMdev    bool   `json:"has_mdev"`    // true if an mdev is created on this VF
+	Allocated  bool   `json:"allocated"`   // true if a vGPU is assigned to this VF
+}
+
+type VGPUDevice struct {
+	Framework   VGPUFramework
+	VFAddress   string
+	ProfileType string
+	ProfileName string
+	SysfsPath   string
+	MdevUUID    string
 }
 
 // MdevDevice represents an active mediated device (vGPU instance)

lib/devices/vgpu_linux.go

@@ -0,0 +1,37 @@
+//go:build linux
+
+package devices
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+)
+
+func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) {
+	mdev, err := CreateMdev(ctx, profileName, instanceID)
+	if err != nil {
+		return nil, err
+	}
+	return &VGPUDevice{
+		Framework:   VGPUFrameworkMdev,
+		VFAddress:   mdev.VFAddress,
+		ProfileType: mdev.ProfileType,
+		ProfileName: mdev.ProfileName,
+		SysfsPath:   mdev.SysfsPath,
+		MdevUUID:    mdev.UUID,
+	}, nil
+}
+
+func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error {
+	if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev {
+		return fmt.Errorf("unknown vGPU framework %q", framework)
+	}
+	if mdevUUID == "" {
+		if devicePath == "" {
+			return nil
+		}
+		mdevUUID = filepath.Base(devicePath)
+	}
+	return DestroyMdev(ctx, mdevUUID)
+}

lib/hypervisor/cloudhypervisor/config.go

@@ -125,10 +125,16 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig {
 	}
 
 	// Device passthrough configuration
+	devicePaths := make([]string, 0, len(cfg.PCIDevices)+1)
+	devicePaths = append(devicePaths, cfg.PCIDevices...)
+	if cfg.VGPUDevicePath != "" {
+		devicePaths = append(devicePaths, cfg.VGPUDevicePath)
+	}
+
 	var devices *[]vmm.DeviceConfig
-	if len(cfg.PCIDevices) > 0 {
-		deviceConfigs := make([]vmm.DeviceConfig, 0, len(cfg.PCIDevices))
-		for _, path := range cfg.PCIDevices {
+	if len(devicePaths) > 0 {
+		deviceConfigs := make([]vmm.DeviceConfig, 0, len(devicePaths))
+		for _, path := range devicePaths {
 			deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{
 				Path: path,
 			})

lib/hypervisor/cloudhypervisor/config_test.go

@@ -8,6 +8,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestToVMConfig_VGPU(t *testing.T) {
+	t.Parallel()
+
+	path := "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123"
+	vmCfg := ToVMConfig(hypervisor.VMConfig{VGPUDevicePath: path})
+
+	require.NotNil(t, vmCfg.Devices)
+	require.Len(t, *vmCfg.Devices, 1)
+	assert.Equal(t, path, (*vmCfg.Devices)[0].Path)
+}
+
 func TestToVMConfig_GuestMemoryBalloon(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/hypervisor/config.go

@@ -24,7 +24,8 @@ type VMConfig struct {
 	VsockSocket string
 
 	// PCI device passthrough (GPU, etc.)
-	PCIDevices []string
+	PCIDevices     []string
+	VGPUDevicePath string
 
 	// Boot configuration
 	KernelPath string

lib/hypervisor/qemu/config.go

@@ -82,13 +82,14 @@ func BuildArgs(cfg hypervisor.VMConfig) []string {
 		args = append(args, "-device", fmt.Sprintf("vhost-vsock-pci,guest-cid=%d", cfg.VsockCID))
 	}
 
-	// PCI device passthrough (GPU, mdev vGPU, etc.)
+	if cfg.VGPUDevicePath != "" {
+		args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath))
+	}
+
+	// Whole-device PCI passthrough (vGPU attaches via VGPUDevicePath above)
 	for _, devicePath := range cfg.PCIDevices {
 		var deviceArg string
-		if strings.HasPrefix(devicePath, "/sys/bus/mdev/devices/") {
-			// mdev device (vGPU) - use sysfsdev parameter
-			deviceArg = fmt.Sprintf("vfio-pci,sysfsdev=%s", devicePath)
-		} else if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") {
+		if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") {
 			// Full sysfs path for regular PCI device - extract the PCI address
 			// Using filepath.Base is more robust than manual string splitting
 			pciAddr := filepath.Base(strings.TrimSuffix(devicePath, "/"))

lib/hypervisor/qemu/config_test.go

@@ -122,6 +122,26 @@ func TestBuildArgs_Vsock(t *testing.T) {
 	assert.Contains(t, args, "vhost-vsock-pci,guest-cid=123")
 }
 
+func TestBuildArgs_VGPU(t *testing.T) {
+	t.Parallel()
+
+	for _, path := range []string{
+		"/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123",
+		"/sys/bus/pci/devices/0000:82:00.4",
+	} {
+		path := path
+		t.Run(path, func(t *testing.T) {
+			t.Parallel()
+			args := BuildArgs(hypervisor.VMConfig{
+				VCPUs:          1,
+				MemoryBytes:    512 * 1024 * 1024,
+				VGPUDevicePath: path,
+			})
+			assert.Contains(t, args, "vfio-pci,sysfsdev="+path)
+		})
+	}
+}
+
 func TestBuildArgs_PCIPassthrough(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/instances/create.go

@@ -52,11 +52,11 @@ var systemDirectories = []string{
 	"/var",
 }
 
-func wrapCreateMdevErr(profile string, err error) error {
+func wrapCreateVGPUErr(profile string, err error) error {
 	if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) {
 		return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
 	}
-	return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err)
+	return fmt.Errorf("create vGPU for profile %s: %w", profile, err)
 }
 
 // generateVsockCID converts first 8 chars of instance ID to a unique CID
@@ -260,7 +260,10 @@ func (m *manager) createInstance(
 	// whatever devices have been attached when cleanup runs.
 	var attachedDeviceIDs []string
 	var resolvedDeviceIDs []string
+	var gpuDevice *devices.VGPUDevice
 	var gpuProfile string
+	var gpuFramework devices.VGPUFramework
+	var gpuDevicePath string
 	var gpuMdevUUID string
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup
@@ -280,23 +283,23 @@ func (m *manager) createInstance(
 		})
 	}
 
-	// Handle vGPU profile request - create mdev device
+	// Handle vGPU profile request
 	if req.GPU != nil && req.GPU.Profile != "" {
-		log.InfoContext(ctx, "creating vGPU mdev", "instance_id", id, "profile", req.GPU.Profile)
-		mdev, err := devices.CreateMdev(ctx, req.GPU.Profile, id)
+		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
+		gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id)
 		if err != nil {
-			log.ErrorContext(ctx, "failed to create mdev", "profile", req.GPU.Profile, "error", err)
-			return nil, wrapCreateMdevErr(req.GPU.Profile, err)
+			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
+			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
 		}
-		gpuProfile = req.GPU.Profile
-		gpuMdevUUID = mdev.UUID
-		log.InfoContext(ctx, "created vGPU mdev", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID)
+		gpuProfile = gpuDevice.ProfileName
+		gpuFramework = gpuDevice.Framework
+		gpuDevicePath = gpuDevice.SysfsPath
+		gpuMdevUUID = gpuDevice.MdevUUID
 
-		// Add mdev cleanup to stack
+		// Add vGPU cleanup to stack
 		cu.Add(func() {
-			log.DebugContext(ctx, "destroying mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID)
-			if err := devices.DestroyMdev(ctx, gpuMdevUUID); err != nil {
-				log.WarnContext(ctx, "failed to destroy mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID, "error", err)
+			if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil {
+				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
 			}
 		})
 	}
@@ -364,6 +367,8 @@ func (m *manager) createInstance(
 		VsockSocket:              vsockSocket,
 		Devices:                  resolvedDeviceIDs,
 		GPUProfile:               gpuProfile,
+		GPUFramework:             gpuFramework,
+		GPUDevicePath:            gpuDevicePath,
 		GPUMdevUUID:              gpuMdevUUID,
 		Entrypoint:               req.Entrypoint,
 		Cmd:                      req.Cmd,
@@ -885,12 +890,6 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima
 		}
 	}
 
-	// Add vGPU mdev device if configured
-	if inst.GPUMdevUUID != "" {
-		mdevPath := filepath.Join("/sys/bus/mdev/devices", inst.GPUMdevUUID)
-		pciDevices = append(pciDevices, mdevPath)
-	}
-
 	// Build topology if available
 	var topology *hypervisor.CPUTopology
 	if hostTopo := calculateGuestTopology(inst.Vcpus, m.hostTopology); hostTopo != nil {
@@ -910,21 +909,22 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima
 	}
 
 	return hypervisor.VMConfig{
-		VCPUs:         inst.Vcpus,
-		MemoryBytes:   inst.Size,
-		HotplugBytes:  inst.HotplugSize,
-		Topology:      topology,
-		GuestMemory:   m.guestMemoryConfig(),
-		Disks:         disks,
-		Networks:      networks,
-		SerialLogPath: m.paths.InstanceAppLog(inst.Id),
-		VsockCID:      inst.VsockCID,
-		VsockSocket:   inst.VsockSocket,
-		PCIDevices:    pciDevices,
-		KernelPath:    kernelPath,
-		InitrdPath:    initrdPath,
-		KernelArgs:    m.kernelArgs(inst.HypervisorType),
-		EnableRosetta: inst.EnableRosetta,
+		VCPUs:          inst.Vcpus,
+		MemoryBytes:    inst.Size,
+		HotplugBytes:   inst.HotplugSize,
+		Topology:       topology,
+		GuestMemory:    m.guestMemoryConfig(),
+		Disks:          disks,
+		Networks:       networks,
+		SerialLogPath:  m.paths.InstanceAppLog(inst.Id),
+		VsockCID:       inst.VsockCID,
+		VsockSocket:    inst.VsockSocket,
+		PCIDevices:     pciDevices,
+		VGPUDevicePath: storedVGPUDevicePath(&inst.StoredMetadata),
+		KernelPath:     kernelPath,
+		InitrdPath:     initrdPath,
+		KernelArgs:     m.kernelArgs(inst.HypervisorType),
+		EnableRosetta:  inst.EnableRosetta,
 	}, nil
 }
 

lib/instances/create_mdev_test.go

@@ -45,7 +45,7 @@ func TestCreateInstanceRejectsUnsupportedVGPUBeforeResourceReservation(t *testin
 	assert.Zero(t, validator.reserveCalls)
 }
 
-func TestWrapCreateMdevErr(t *testing.T) {
+func TestWrapCreateVGPUErr(t *testing.T) {
 	t.Parallel()
 
 	for _, tc := range []struct {
@@ -61,15 +61,15 @@ func TestWrapCreateMdevErr(t *testing.T) {
 			wantInvalidRequest: true,
 		},
 		{
-			name:        "other mdev error",
+			name:        "other vGPU error",
 			err:         errors.New("boom"),
-			wantMessage: "create vGPU mdev for profile profile: boom",
+			wantMessage: "create vGPU for profile profile: boom",
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			err := wrapCreateMdevErr("profile", tc.err)
+			err := wrapCreateVGPUErr("profile", tc.err)
 
 			assert.ErrorIs(t, err, tc.err)
 			if tc.wantInvalidRequest {