Merge pull request #8546 from zhuchenwang/main

Add Vsock support to KubeVirt

api/api-rule-violations-known.list

@@ -289,6 +289,7 @@ API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceGu
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceMigrationState,MigrationUID
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceNetworkInterface,IP
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceNetworkInterface,IPs
+API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceStatus,VSOCKCID
 API rule violation: names_match,kubevirt.io/api/core/v1,WatchdogDevice,I6300ESB
 API rule violation: names_match,kubevirt.io/api/instancetype/v1alpha1,VirtualMachineInstancetypeSpec,GPUs
 API rule violation: names_match,kubevirt.io/api/instancetype/v1alpha2,VirtualMachineInstancetypeSpec,GPUs

api/api-rule-violations.list

@@ -289,6 +289,7 @@ API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceGu
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceMigrationState,MigrationUID
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceNetworkInterface,IP
 API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceNetworkInterface,IPs
+API rule violation: names_match,kubevirt.io/api/core/v1,VirtualMachineInstanceStatus,VSOCKCID
 API rule violation: names_match,kubevirt.io/api/core/v1,WatchdogDevice,I6300ESB
 API rule violation: names_match,kubevirt.io/api/instancetype/v1alpha1,VirtualMachineInstancetypeSpec,GPUs
 API rule violation: names_match,kubevirt.io/api/instancetype/v1alpha2,VirtualMachineInstancetypeSpec,GPUs

api/openapi-spec/swagger.json

@@ -15746,6 +15746,10 @@
       "description": "Whether to attach the default serial console or not. Serial console access will not be available if set to false. Defaults to true.",
       "type": "boolean"
      },
+     "autoattachVSOCK": {
+      "description": "Whether to attach the VSOCK CID to the VM or not. VSOCK access will be available if set to true. Defaults to false.",
+      "type": "boolean"
+     },
      "blockMultiQueue": {
       "description": "Whether or not to enable virtio multi-queue for block devices. Defaults to false.",
       "type": "boolean"
@@ -19006,6 +19010,11 @@
     "type": "object",
     "nullable": true,
     "properties": {
+     "VSOCKCID": {
+      "description": "VSOCKCID is used to track the allocated VSOCK CID in the VM.",
+      "type": "integer",
+      "format": "int64"
+     },
      "activePods": {
       "description": "ActivePods is a mapping of pod UID to node name. It is possible for multiple pods to be running for a single VMI during migration.",
       "type": "object",

pkg/util/util.go

@@ -136,6 +136,10 @@ func NeedTunDevice(vmi *v1.VirtualMachineInstance) bool {
 		(*vmi.Spec.Domain.Devices.AutoattachPodInterface == true)
 }
 
+func IsAutoAttachVSOCK(vmi *v1.VirtualMachineInstance) bool {
+	return vmi.Spec.Domain.Devices.AutoattachVSOCK != nil && *vmi.Spec.Domain.Devices.AutoattachVSOCK
+}
+
 // UseSoftwareEmulationForDevice determines whether to fallback to software emulation for the given device.
 // This happens when the given device doesn't exist, and software emulation is enabled.
 func UseSoftwareEmulationForDevice(devicePath string, allowEmulation bool) (bool, error) {

pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter.go

@@ -212,6 +212,7 @@ func ValidateVirtualMachineInstanceSpec(field *k8sfield.Path, spec *v1.VirtualMa
 	causes = append(causes, validateHostDevicesWithPassthroughEnabled(field, spec, config)...)
 	causes = append(causes, validateSoundDevices(field, spec)...)
 	causes = append(causes, validateLaunchSecurity(field, spec, config)...)
+	causes = append(causes, validateVSOCK(field, spec, config)...)
 
 	return causes
 }
@@ -2530,3 +2531,19 @@ func validateSpecTopologySpreadConstraints(field *k8sfield.Path, spec *v1.Virtua
 
 	return
 }
+
+func validateVSOCK(field *k8sfield.Path, spec *v1.VirtualMachineInstanceSpec, config *virtconfig.ClusterConfig) (causes []metav1.StatusCause) {
+	if spec.Domain.Devices.AutoattachVSOCK == nil || !*spec.Domain.Devices.AutoattachVSOCK {
+		return
+	}
+
+	if !config.VSOCKEnabled() {
+		causes = append(causes, metav1.StatusCause{
+			Type:    metav1.CauseTypeFieldValueInvalid,
+			Message: fmt.Sprintf("%s feature gate is not enabled in kubevirt-config", virtconfig.VSOCKGate),
+			Field:   field.Child("domain", "devices", "autoattachVSOCK").String(),
+		})
+	}
+
+	return
+}

pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter_test.go

@@ -3739,6 +3739,34 @@ var _ = Describe("Validating VMICreate Admitter", func() {
 		})
 	})
 
+	Context("with vsocks defined", func() {
+		var vmi *v1.VirtualMachineInstance
+		BeforeEach(func() {
+			vmi = api.NewMinimalVMI("testvmi")
+			enableFeatureGate(virtconfig.VSOCKGate)
+		})
+		Context("feature gate enabled", func() {
+			It("should accept vmi with no vsocks defined", func() {
+				causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+				Expect(causes).To(HaveLen(0))
+			})
+			It("should accept vmi with vsocks defined", func() {
+				vmi.Spec.Domain.Devices.AutoattachVSOCK = pointer.Bool(true)
+				causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+				Expect(causes).To(HaveLen(0))
+			})
+		})
+		Context("feature gate disabled", func() {
+			It("should reject when the feature gate is disabled", func() {
+				disableFeatureGates()
+				vmi.Spec.Domain.Devices.AutoattachVSOCK = pointer.Bool(true)
+				causes := ValidateVirtualMachineInstanceSpec(k8sfield.NewPath("fake"), &vmi.Spec, config)
+				Expect(causes).To(HaveLen(1))
+				Expect(causes[0].Message).To(ContainSubstring(fmt.Sprintf("%s feature gate is not enabled", virtconfig.VSOCKGate)))
+			})
+		})
+	})
+
 	Context("with affinity checks", func() {
 		var vmi *v1.VirtualMachineInstance
 		BeforeEach(func() {

pkg/virt-config/feature-gates.go

@@ -51,6 +51,7 @@ const (
 	// DockerSELinuxMCSWorkaround sets the SELinux level of all the non-compute virt-launcher containers to "s0".
 	DockerSELinuxMCSWorkaround = "DockerSELinuxMCSWorkaround"
 	PSA                        = "PSA"
+	VSOCKGate                  = "VSOCK"
 )
 
 var deprecatedFeatureGates = [...]string{
@@ -178,3 +179,7 @@ func (config *ClusterConfig) DockerSELinuxMCSWorkaroundEnabled() bool {
 func (config *ClusterConfig) PSAEnabled() bool {
 	return config.isFeatureGateEnabled(PSA)
 }
+
+func (config *ClusterConfig) VSOCKEnabled() bool {
+	return config.isFeatureGateEnabled(VSOCKGate)
+}

pkg/virt-controller/services/BUILD.bazel

@@ -86,5 +86,6 @@ go_test(
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/utils/pointer:go_default_library",
     ],
 )

pkg/virt-controller/services/renderresources.go

@@ -264,11 +264,11 @@ func copyResources(srcResources, dstResources k8sv1.ResourceList) {
 // memory needed for the domain to operate properly.
 // This includes the memory needed for the guest and memory
 // for Qemu and OS overhead.
-//
 // The return value is overhead memory quantity
 //
 // Note: This is the best estimation we were able to come up with
-//       and is still not 100% accurate
+//
+//	and is still not 100% accurate
 func GetMemoryOverhead(vmi *v1.VirtualMachineInstance, cpuArch string) *resource.Quantity {
 	domain := vmi.Spec.Domain
 	vmiMemoryReq := domain.Resources.Requests.Memory()
@@ -423,6 +423,9 @@ func getRequiredResources(vmi *v1.VirtualMachineInstance, allowEmulation bool) k
 	if !allowEmulation {
 		res[KvmDevice] = resource.MustParse("1")
 	}
+	if util.IsAutoAttachVSOCK(vmi) {
+		res[VhostVsockDevice] = resource.MustParse("1")
+	}
 	return res
 }
 

pkg/virt-controller/services/template.go

@@ -66,6 +66,7 @@ const KvmDevice = "devices.kubevirt.io/kvm"
 const TunDevice = "devices.kubevirt.io/tun"
 const VhostNetDevice = "devices.kubevirt.io/vhost-net"
 const SevDevice = "devices.kubevirt.io/sev"
+const VhostVsockDevice = "devices.kubevirt.io/vhost-vsock"
 
 const debugLogs = "debugLogs"
 const logVerbosity = "logVerbosity"

pkg/virt-controller/services/template_test.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/utils/pointer"
 
 	"kubevirt.io/client-go/api"
 
@@ -3554,6 +3555,18 @@ var _ = Describe("Template", func() {
 			Expect(int(sev.Value())).To(Equal(1))
 		})
 	})
+
+	Context("with VSOCK enabled", func() {
+		It("should add VSOCK device to resources", func() {
+			vmi := api.NewMinimalVMI("fake-vmi")
+			vmi.Spec.Domain.Devices.AutoattachVSOCK = pointer.Bool(true)
+
+			pod, err := svc.RenderLaunchManifest(vmi)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(pod).ToNot(BeNil())
+			Expect(pod.Spec.Containers[0].Resources.Limits).To(HaveKey(kubev1.ResourceName(VhostVsockDevice)))
+		})
+	})
 })
 
 var _ = Describe("getResourceNameForNetwork", func() {

pkg/virt-controller/watch/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "util.go",
         "vm.go",
         "vmi.go",
+        "vsock.go",
     ],
     importpath = "kubevirt.io/kubevirt/pkg/virt-controller/watch",
     visibility = ["//visibility:public"],
@@ -111,6 +112,7 @@ go_test(
         "replicaset_test.go",
         "vm_test.go",
         "vmi_test.go",
+        "vsock_test.go",
         "watch_suite_test.go",
     ],
     embed = [":go_default_library"],

pkg/virt-controller/watch/vmi.go

@@ -48,6 +48,7 @@ import (
 	"kubevirt.io/kubevirt/pkg/network/sriov"
 	"kubevirt.io/kubevirt/pkg/network/vmispec"
 	storagetypes "kubevirt.io/kubevirt/pkg/storage/types"
+	"kubevirt.io/kubevirt/pkg/util"
 	traceUtils "kubevirt.io/kubevirt/pkg/util/trace"
 	patchtypes "kubevirt.io/kubevirt/pkg/util/types"
 	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
@@ -155,6 +156,7 @@ func NewVMIController(templateService services.TemplateService,
 		topologyHinter:     topologyHinter,
 		namespaceStore:     namespaceStore,
 		onOpenshift:        onOpenshift,
+		cidsMap:            newCIDsMap(),
 	}
 
 	c.vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -218,6 +220,7 @@ type VMIController struct {
 	clusterConfig      *virtconfig.ClusterConfig
 	namespaceStore     cache.Store
 	onOpenshift        bool
+	cidsMap            *cidsMap
 }
 
 func (c *VMIController) Run(threadiness int, stopCh <-chan struct{}) {
@@ -227,6 +230,13 @@ func (c *VMIController) Run(threadiness int, stopCh <-chan struct{}) {
 
 	// Wait for cache sync before we start the pod controller
 	cache.WaitForCacheSync(stopCh, c.vmInformer.HasSynced, c.vmiInformer.HasSynced, c.podInformer.HasSynced, c.dataVolumeInformer.HasSynced, c.cdiConfigInformer.HasSynced, c.cdiInformer.HasSynced)
+	// Sync the CIDs from exist VMIs
+	var vmis []*virtv1.VirtualMachineInstance
+	for _, obj := range c.vmiInformer.GetStore().List() {
+		vmi := obj.(*virtv1.VirtualMachineInstance)
+		vmis = append(vmis, vmi)
+	}
+	c.cidsMap.Sync(vmis)
 
 	// Start the actual work
 	for i := 0; i < threadiness; i++ {
@@ -279,6 +289,7 @@ func (c *VMIController) execute(key string) error {
 	if !exists {
 		c.podExpectations.DeleteExpectations(key)
 		c.vmiExpectations.DeleteExpectations(key)
+		c.cidsMap.Remove(key)
 		return nil
 	}
 	vmi := obj.(*virtv1.VirtualMachineInstance)
@@ -550,6 +561,13 @@ func (c *VMIController) updateStatus(vmi *virtv1.VirtualMachineInstance, pod *k8
 				if shouldSetMigrationTransport(pod) {
 					vmiCopy.Status.MigrationTransport = virtv1.MigrationTransportUnix
 				}
+
+				// Allocate the CID if VSOCK is enabled.
+				if util.IsAutoAttachVSOCK(vmiCopy) {
+					if err := c.cidsMap.Allocate(vmiCopy); err != nil {
+						return err
+					}
+				}
 			} else if isPodDownOrGoingDown(pod) {
 				vmiCopy.Status.Phase = virtv1.Failed
 			}

pkg/virt-controller/watch/vmi_test.go

@@ -2940,6 +2940,41 @@ var _ = Describe("VirtualMachineInstance watcher", func() {
 
 		})
 	})
+
+	Context("auto attach VSOCK", func() {
+		It("should allocate CID when VirtualMachineInstance is scheduled", func() {
+			vmi := NewPendingVirtualMachine("testvmi")
+			setReadyCondition(vmi, k8sv1.ConditionFalse, virtv1.GuestNotRunningReason)
+			vmi.Status.Phase = virtv1.Scheduling
+			vmi.Spec.Domain.Devices.AutoattachVSOCK = pointer.Bool(true)
+			pod := NewPodForVirtualMachine(vmi, k8sv1.PodRunning)
+
+			addVirtualMachine(vmi)
+			podFeeder.Add(pod)
+
+			vmiInterface.EXPECT().Update(gomock.Any()).Do(func(arg *virtv1.VirtualMachineInstance) {
+				Expect(arg.Status.Phase).To(Equal(virtv1.Scheduled))
+				Expect(arg.Status.PhaseTransitionTimestamps).ToNot(BeEmpty())
+				Expect(arg.Status.PhaseTransitionTimestamps).ToNot(HaveLen(len(vmi.Status.PhaseTransitionTimestamps)))
+				Expect(arg.Status.VSOCKCID).NotTo(BeNil())
+			}).Return(vmi, nil)
+			controller.Execute()
+		})
+
+		It("should recycle the CID when the pods are deleted", func() {
+			vmi := NewPendingVirtualMachine("testvmi")
+			vmi.Spec.Domain.Devices.AutoattachVSOCK = pointer.Bool(true)
+			Expect(controller.cidsMap.Allocate(vmi)).To(Succeed())
+			vmi.Status.Phase = virtv1.Succeeded
+			addVirtualMachine(vmi)
+
+			Expect(vmiInformer.GetIndexer().Delete(vmi)).To(Succeed())
+			controller.Execute()
+
+			Expect(controller.cidsMap.cids).To(BeEmpty())
+			Expect(controller.cidsMap.reverse).To(BeEmpty())
+		})
+	})
 })
 
 func NewDv(namespace string, name string, phase cdiv1.DataVolumePhase) *cdiv1.DataVolume {