Handle qemu pool backend switches and version policy

Author: cursoragent

lib/hypervisor/qemu/pool.go

@@ -1,7 +1,6 @@
 package qemu
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/kernel/hypeman/lib/hypervisor"
@@ -28,11 +27,11 @@ func GetOrCreateForType(socketPath string, hypervisorType hypervisor.Type) (*QEM
 	// Try read lock first for existing connection
 	clientPool.RLock()
 	if client, ok := clientPool.clients[socketPath]; ok {
-		clientPool.RUnlock()
-		if client.hypervisorType != hypervisorType {
-			return nil, poolTypeMismatchError(socketPath, client.hypervisorType, hypervisorType)
+		if client.hypervisorType == hypervisorType {
+			clientPool.RUnlock()
+			return client, nil
 		}
-		return client, nil
+		// Backend identity changed for this socket path. Recreate under write lock.
 	}
 	clientPool.RUnlock()
 
@@ -42,10 +41,11 @@ func GetOrCreateForType(socketPath string, hypervisorType hypervisor.Type) (*QEM
 
 	// Double-check after acquiring write lock
 	if client, ok := clientPool.clients[socketPath]; ok {
-		if client.hypervisorType != hypervisorType {
-			return nil, poolTypeMismatchError(socketPath, client.hypervisorType, hypervisorType)
+		if client.hypervisorType == hypervisorType {
+			return client, nil
 		}
-		return client, nil
+		// Stale pooled backend type for this socket path. Drop and reconnect.
+		removeLocked(socketPath)
 	}
 
 	// Create new client
@@ -58,20 +58,22 @@ func GetOrCreateForType(socketPath string, hypervisorType hypervisor.Type) (*QEM
 	return client, nil
 }
 
-func poolTypeMismatchError(socketPath string, cached, requested hypervisor.Type) error {
-	return fmt.Errorf("QEMU client for %s is pooled as hypervisor %s, not %s", socketPath, cached, requested)
-}
-
 // Remove closes and removes a client from the pool.
 // Called automatically on errors to allow fresh reconnection.
 // Close is done asynchronously to avoid blocking if the connection is in a bad state.
 func Remove(socketPath string) {
 	clientPool.Lock()
 	defer clientPool.Unlock()
+	removeLocked(socketPath)
+}
 
+// removeLocked removes an entry while clientPool lock is held.
+func removeLocked(socketPath string) {
 	if client, ok := clientPool.clients[socketPath]; ok {
 		delete(clientPool.clients, socketPath)
 		// Close asynchronously to avoid blocking on stuck connections
-		go client.client.Close()
+		if client.client != nil {
+			go client.client.Close()
+		}
 	}
 }

lib/hypervisor/qemu/pool_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetOrCreateForTypeRejectsCachedBackendMismatch(t *testing.T) {
+func TestGetOrCreateForTypeEvictsCachedBackendMismatch(t *testing.T) {
 	socketPath := t.TempDir() + "/qemu.sock"
 	clientPool.Lock()
 	clientPool.clients[socketPath] = &QEMU{socketPath: socketPath, hypervisorType: hypervisor.TypeQEMU}
@@ -19,5 +19,9 @@ func TestGetOrCreateForTypeRejectsCachedBackendMismatch(t *testing.T) {
 	})
 
 	_, err := GetOrCreateForType(socketPath, hypervisor.TypeQEMUMicroVM)
-	require.ErrorContains(t, err, "pooled as hypervisor qemu, not qemu-microvm")
+	require.Error(t, err)
+	clientPool.RLock()
+	_, stillPooled := clientPool.clients[socketPath]
+	clientPool.RUnlock()
+	require.False(t, stillPooled, "stale cached backend must be evicted after mismatch")
 }

lib/instances/create.go

@@ -748,13 +748,13 @@ func (m *manager) startAndBootVM(
 		return fmt.Errorf("get vm starter: %w", err)
 	}
 
-	// qemu-microvm snapshots are tied to the binary that boots the VM. Refresh
-	// metadata on every cold start so host upgrades do not leave the instance's
-	// reported version pinned to its original creation time.
-	if stored.HypervisorType == hypervisor.TypeQEMUMicroVM {
+	// Some hypervisors tie snapshot compatibility to the exact runtime binary.
+	// Refresh metadata on cold start so host upgrades do not leave the stored
+	// version pinned to original creation time.
+	if refreshHypervisorVersionOnColdStart(stored.HypervisorType) {
 		detectedVersion, err := starter.GetVersion(m.paths)
 		if err != nil {
-			return fmt.Errorf("get QEMU version for qemu-microvm start: %w", err)
+			return fmt.Errorf("get hypervisor version for %s start: %w", stored.HypervisorType, err)
 		}
 		stored.HypervisorVersion = detectedVersion
 	}

lib/instances/hypervisor_version.go

@@ -8,19 +8,46 @@ import (
 	"github.com/kernel/hypeman/lib/logger"
 )
 
+type hypervisorVersionPolicy struct {
+	enforceExactInstalledVersion bool
+	refreshOnColdStart           bool
+}
+
+var hypervisorVersionPoliciesByType = map[hypervisor.Type]hypervisorVersionPolicy{
+	hypervisor.TypeQEMUMicroVM: {
+		enforceExactInstalledVersion: true,
+		refreshOnColdStart:           true,
+	},
+}
+
+func resolveHypervisorVersionPolicy(hvType hypervisor.Type) hypervisorVersionPolicy {
+	if policy, ok := hypervisorVersionPoliciesByType[hvType]; ok {
+		return policy
+	}
+	return hypervisorVersionPolicy{}
+}
+
+func enforceExactInstalledHypervisorVersion(hvType hypervisor.Type) bool {
+	return resolveHypervisorVersionPolicy(hvType).enforceExactInstalledVersion
+}
+
+func refreshHypervisorVersionOnColdStart(hvType hypervisor.Type) bool {
+	return resolveHypervisorVersionPolicy(hvType).refreshOnColdStart
+}
+
 func (m *manager) resolveCreateHypervisorVersion(
 	ctx context.Context,
 	starter hypervisor.VMStarter,
 	hvType hypervisor.Type,
 	requested string,
 ) (string, error) {
-	if hvType == hypervisor.TypeQEMUMicroVM {
+	if enforceExactInstalledHypervisorVersion(hvType) {
 		detected, err := starter.GetVersion(m.paths)
 		if err != nil {
-			return "", fmt.Errorf("get QEMU version for qemu-microvm: %w", err)
+			return "", fmt.Errorf("get installed hypervisor version for %s: %w", hvType, err)
 		}
 		if requested != "" && requested != detected {
-			return "", fmt.Errorf("%w: requested qemu-microvm version %q does not match installed QEMU %q", ErrInvalidRequest, requested, detected)
+			return "", fmt.Errorf("%w: requested %s hypervisor version %q does not match installed version %q", ErrInvalidRequest, hvType, requested, detected)
 		}
 		return detected, nil
 	}

lib/instances/snapshot.go

@@ -532,8 +532,8 @@ func (m *manager) prepareSnapshotTarget(ctx context.Context, source StoredMetada
 	if target != source.HypervisorType {
 		version, err = starter.GetVersion(m.paths)
 		if err != nil {
-			if target == hypervisor.TypeQEMUMicroVM {
-				return nil, "", fmt.Errorf("get QEMU version for qemu-microvm snapshot target: %w", err)
+			if enforceExactInstalledHypervisorVersion(target) {
+				return nil, "", fmt.Errorf("get installed hypervisor version for %s snapshot target: %w", target, err)
 			}
 			logger.FromContext(ctx).WarnContext(ctx, "failed to get hypervisor version", "hypervisor", target, "error", err)
 			version = "unknown"