Add kubevirt.io/realtime node label and logic

Signed-off-by: Jordi Gil <jgil@redhat.com>

pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator.go

@@ -116,6 +116,11 @@ func (mutator *VMIsMutator) Mutate(ar *admissionv1.AdmissionReview) *admissionv1
 				log.Log.V(2).Infof("Failed to setting for Arm64: %s", err)
 			}
 		}
+		if newVMI.IsRealtimeEnabled() {
+			log.Log.V(4).Info("Add realtime node label selector")
+			addRealtimeNodeSelector(newVMI)
+		}
+
 		// Add foreground finalizer
 		newVMI.Finalizers = append(newVMI.Finalizers, v1.VirtualMachineInstanceFinalizer)
 
@@ -356,3 +361,11 @@ func canBeNonRoot(vmi *v1.VirtualMachineInstance) error {
 	}
 	return nil
 }
+
+// AddRealtimeNodeSelector adds the realtime node selector
+func addRealtimeNodeSelector(vmi *v1.VirtualMachineInstance) {
+	if vmi.Spec.NodeSelector == nil {
+		vmi.Spec.NodeSelector = map[string]string{}
+	}
+	vmi.Spec.NodeSelector[v1.RealtimeLabel] = ""
+}

pkg/virt-api/webhooks/mutating-webhook/mutators/vmi-mutator_test.go

@@ -941,4 +941,22 @@ var _ = Describe("VirtualMachineInstance Mutator", func() {
 		})
 	})
 
+	It("should add realtime node label selector with realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: &v1.Realtime{}}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).NotTo(BeNil())
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.RealtimeLabel: ""}))
+	})
+	It("should not add realtime node label selector when no realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: nil}
+		vmi.Spec.NodeSelector = map[string]string{v1.NodeSchedulable: "true"}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.NodeSchedulable: "true"}))
+	})
+	It("should not overwrite existing node label selectors with realtime workload", func() {
+		vmi.Spec.Domain.CPU = &v1.CPU{Realtime: &v1.Realtime{}}
+		vmi.Spec.NodeSelector = map[string]string{v1.NodeSchedulable: "true"}
+		vmiSpec, _ := getVMISpecMetaFromResponse()
+		Expect(vmiSpec.NodeSelector).To(BeEquivalentTo(map[string]string{v1.NodeSchedulable: "true", v1.RealtimeLabel: ""}))
+	})
 })

pkg/virt-handler/node-labeller/node_labeller.go

@@ -23,6 +23,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os/exec"
 	"reflect"
 	"strings"
 	"time"
@@ -266,6 +267,14 @@ func (n *NodeLabeller) prepareLabels(cpuModels []string, cpuFeatures cpuFeatures
 	newLabels[kubevirtv1.CPUModelVendorLabel+n.cpuModelVendor] = "true"
 	newLabels[kubevirtv1.HostModelCPULabel+hostCpuModel.name] = "true"
 
+	capable, err := isNodeRealtimeCapable()
+	if err != nil {
+		n.logger.Reason(err).Error("failed to identify if a node is capable of running realtime workloads")
+	}
+	if capable {
+		newLabels[kubevirtv1.RealtimeLabel] = ""
+	}
+
 	return newLabels
 }
 
@@ -290,7 +299,8 @@ func (n *NodeLabeller) removeLabellerLabels(node *v1.Node) {
 			strings.Contains(label, kubevirtv1.CPUFeatureLabel) ||
 			strings.Contains(label, kubevirtv1.CPUModelLabel) ||
 			strings.Contains(label, kubevirtv1.CPUTimerLabel) ||
-			strings.Contains(label, kubevirtv1.HypervLabel) {
+			strings.Contains(label, kubevirtv1.HypervLabel) ||
+			strings.Contains(label, kubevirtv1.RealtimeLabel) {
 			delete(node.Labels, label)
 		}
 	}
@@ -301,3 +311,19 @@ func (n *NodeLabeller) removeLabellerLabels(node *v1.Node) {
 		}
 	}
 }
+
+const kernelSchedRealtimeRuntimeInMicrosecods = "kernel.sched_rt_runtime_us"
+
+// isNodeRealtimeCapable Checks if a node is capable of running realtime workloads. Currently by validating if the kernel system setting value
+// for `kernel.sched_rt_runtime_us` is set to allow running realtime scheduling with unlimited time (==-1)
+// TODO: This part should be improved to validate against key attributes that determine best if a host is able to run realtime
+// workloads at peak performance.
+
+func isNodeRealtimeCapable() (bool, error) {
+	ret, err := exec.Command("sysctl", kernelSchedRealtimeRuntimeInMicrosecods).CombinedOutput()
+	if err != nil {
+		return false, err
+	}
+	st := strings.Trim(string(ret), "\n")
+	return fmt.Sprintf("%s = -1", kernelSchedRealtimeRuntimeInMicrosecods) == st, nil
+}

staging/src/kubevirt.io/client-go/api/v1/types.go

@@ -784,6 +784,9 @@ const (
 
 	// MigrationTransportUnixAnnotation means that the VMI will be migrated using the unix URI
 	MigrationTransportUnixAnnotation string = "kubevirt.io/migrationTransportUnix"
+
+	// RealtimeLabel marks the node as capable of running realtime workloads
+	RealtimeLabel string = "kubevirt.io/realtime"
 )
 
 func NewVMI(name string, uid types.UID) *VirtualMachineInstance {

tests/framework/checks/checks.go

@@ -23,6 +23,16 @@ func IsCPUManagerPresent(node *v1.Node) bool {
 	return nodeHaveCpuManagerLabel
 }
 
+func IsRealtimeCapable(node *v1.Node) bool {
+	gomega.Expect(node).ToNot(gomega.BeNil())
+	for label, _ := range node.Labels {
+		if label == v12.RealtimeLabel {
+			return true
+		}
+	}
+	return false
+}
+
 func Has2MiHugepages(node *v1.Node) bool {
 	gomega.Expect(node).ToNot(gomega.BeNil())
 	_, exists := node.Status.Capacity[v1.ResourceHugePagesPrefix+"2Mi"]

tests/framework/checks/skips.go

@@ -49,3 +49,18 @@ func SkipTestIfNoCPUManagerWith2MiHugepages() {
 	}
 	ginkgo.Skip("no node with CPUManager and 2Mi hugepages detected", 1)
 }
+
+func SkipTestIfNotRealtimeCapable() {
+
+	virtClient, err := kubecli.GetKubevirtClient()
+	util.PanicOnError(err)
+	nodes := util.GetAllSchedulableNodes(virtClient)
+
+	for _, node := range nodes.Items {
+		if IsRealtimeCapable(&node) && IsCPUManagerPresent(&node) && Has2MiHugepages(&node) {
+			return
+		}
+	}
+	ginkgo.Skip("no node capable of running realtime workloads detected", 1)
+
+}

tests/realtime/realtime.go

@@ -74,7 +74,7 @@ var _ = Describe("[sig-compute-realtime][Serial]Realtime", func() {
 		Expect(err).ToNot(HaveOccurred())
 		checks.SkipTestIfNoFeatureGate(virtconfig.NUMAFeatureGate)
 		checks.SkipTestIfNoFeatureGate(virtconfig.CPUManager)
-		checks.SkipTestIfNoCPUManagerWith2MiHugepages()
+		checks.SkipTestIfNotRealtimeCapable()
 		tests.BeforeTestCleanup()
 
 	})