Skip to content

Commit

Permalink
Collect VMI OS info from the Guest agent
Browse files Browse the repository at this point in the history 
Signed-off-by: assafad <aadmi@redhat.com>
  • Loading branch information
@assafad
assafad committed Apr 9, 2024
1 parent 4933164 commit 7170053
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 37 deletions.
64 changes: 47 additions & 17 deletions pkg/monitoring/metrics/virt-controller/vmistats_collector.go
View file
Expand Up @@ -61,7 +61,7 @@ var (
Name: "kubevirt_vmi_phase_count",
Help: "Sum of VMIs per phase and node. `phase` can be one of the following: [`Pending`, `Scheduling`, `Scheduled`, `Running`, `Succeeded`, `Failed`, `Unknown`].",
},
[]string{"node", "phase", "os", "workload", "flavor", "instance_type", "preference"},
[]string{"node", "phase", "os", "workload", "flavor", "instance_type", "preference", "guest_os_kernel_release", "guest_os_machine", "guest_os_name", "guest_os_version_id"},
)

vmiEvictionBlocker = operatormetrics.NewGaugeVec(
Expand All @@ -74,13 +74,17 @@ var (
)

type vmiCountMetric struct {
Phase string
OS string
Workload string
Flavor string
InstanceType string
Preference string
NodeName string
Phase string
OS string
Workload string
Flavor string
InstanceType string
Preference string
NodeName string
GuestOSKernelRelease string
GuestOSMachine string
GuestOSName string
GuestOSVersionID string
}

func vmiStatsCollectorCallback() []operatormetrics.CollectorResult {
Expand Down Expand Up @@ -117,8 +121,9 @@ func getVmisPhase(vmis []*k6tv1.VirtualMachineInstance) []operatormetrics.Collec
for vmc, count := range countMap {
cr = append(cr, operatormetrics.CollectorResult{
Metric: vmiCount,
Labels: []string{vmc.NodeName, vmc.Phase, vmc.OS, vmc.Workload, vmc.Flavor, vmc.InstanceType, vmc.Preference},
Value: float64(count),
Labels: []string{vmc.NodeName, vmc.Phase, vmc.OS, vmc.Workload, vmc.Flavor,
vmc.InstanceType, vmc.Preference, vmc.GuestOSKernelRelease, vmc.GuestOSMachine, vmc.GuestOSName, vmc.GuestOSVersionID},
Value: float64(count),
})
}

Expand All @@ -137,16 +142,21 @@ func makeVMICountMetricMap(vmis []*k6tv1.VirtualMachineInstance) map[vmiCountMet

func newVMICountMetric(vmi *k6tv1.VirtualMachineInstance) vmiCountMetric {
vmc := vmiCountMetric{
Phase: strings.ToLower(string(vmi.Status.Phase)),
OS: none,
Workload: none,
Flavor: none,
InstanceType: none,
Preference: none,
NodeName: vmi.Status.NodeName,
Phase: strings.ToLower(string(vmi.Status.Phase)),
OS: none,
Workload: none,
Flavor: none,
InstanceType: none,
Preference: none,
GuestOSKernelRelease: none,
GuestOSMachine: none,
GuestOSName: none,
GuestOSVersionID: none,
NodeName: vmi.Status.NodeName,
}

updateFromAnnotations(&vmc, vmi.Annotations)
updateFromGuestOSInfo(&vmc, vmi.Status.GuestOSInfo)

return vmc
}
Expand Down Expand Up @@ -228,6 +238,26 @@ func setPreferenceFromAnnotations(vmc *vmiCountMetric, annotations map[string]st
}
}

func updateFromGuestOSInfo(vmc *vmiCountMetric, guestOSInfo k6tv1.VirtualMachineInstanceGuestOSInfo) {
if guestOSInfo != (k6tv1.VirtualMachineInstanceGuestOSInfo{}) {
if guestOSInfo.KernelRelease != "" {
vmc.GuestOSKernelRelease = guestOSInfo.KernelRelease
}

if guestOSInfo.Machine != "" {
vmc.GuestOSMachine = guestOSInfo.Machine
}

if guestOSInfo.Name != "" {
vmc.GuestOSName = guestOSInfo.Name
}

if guestOSInfo.VersionID != "" {
vmc.GuestOSVersionID = guestOSInfo.VersionID
}
}
}

func getEvictionBlocker(vmis []*k6tv1.VirtualMachineInstance) []operatormetrics.CollectorResult {
var cr []operatormetrics.CollectorResult

Expand Down
58 changes: 38 additions & 20 deletions pkg/monitoring/metrics/virt-controller/vmistats_collector_test.go
View file
Expand Up @@ -152,6 +152,12 @@ var _ = Describe("Utility functions", func() {
},
Status: k6tv1.VirtualMachineInstanceStatus{
Phase: "Pending",
GuestOSInfo: k6tv1.VirtualMachineInstanceGuestOSInfo{
KernelRelease: "6.5.6-300.fc39.x86_64",
Machine: "x86_64",
Name: "Fedora Linux",
VersionID: "39",
},
},
},
{
Expand Down Expand Up @@ -189,28 +195,40 @@ var _ = Describe("Utility functions", func() {
Expect(countMap).To(HaveLen(3))

running := vmiCountMetric{
Phase: "running",
OS: "centos8",
Workload: "server",
Flavor: "tiny",
InstanceType: "<none>",
Preference: "<none>",
Phase: "running",
OS: "centos8",
Workload: "server",
Flavor: "tiny",
GuestOSKernelRelease: "<none>",
GuestOSMachine: "<none>",
GuestOSName: "<none>",
GuestOSVersionID: "<none>",
InstanceType: "<none>",
Preference: "<none>",
}
pending := vmiCountMetric{
Phase: "pending",
OS: "fedora33",
Workload: "workstation",
Flavor: "large",
InstanceType: "<none>",
Preference: "<none>",
Phase: "pending",
OS: "fedora33",
Workload: "workstation",
Flavor: "large",
InstanceType: "<none>",
Preference: "<none>",
GuestOSKernelRelease: "6.5.6-300.fc39.x86_64",
GuestOSMachine: "x86_64",
GuestOSName: "Fedora Linux",
GuestOSVersionID: "39",
}
scheduling := vmiCountMetric{
Phase: "scheduling",
OS: "centos7",
Workload: "server",
Flavor: "medium",
InstanceType: "<none>",
Preference: "<none>",
Phase: "scheduling",
OS: "centos7",
Workload: "server",
Flavor: "medium",
GuestOSKernelRelease: "<none>",
GuestOSMachine: "<none>",
GuestOSName: "<none>",
GuestOSVersionID: "<none>",
InstanceType: "<none>",
Preference: "<none>",
}
bogus := vmiCountMetric{
Phase: "bogus",
Expand Down Expand Up @@ -243,7 +261,7 @@ var _ = Describe("Utility functions", func() {
Expect(phaseResultMetric).ToNot(BeNil())
Expect(phaseResultMetric.Metric.GetOpts().Name).To(ContainSubstring("kubevirt_vmi_phase_count"))
Expect(phaseResultMetric.Value).To(BeEquivalentTo(1))
Expect(phaseResultMetric.Labels).To(HaveLen(7))
Expect(phaseResultMetric.Labels).To(HaveLen(11))
Expect(phaseResultMetric.Labels[5]).To(Equal(expected))
},
Entry("with no instance type expect <none>", k6tv1.InstancetypeAnnotation, "", "<none>"),
Expand Down Expand Up @@ -276,7 +294,7 @@ var _ = Describe("Utility functions", func() {

Expect(phaseResultMetric.Metric.GetOpts().Name).To(ContainSubstring("kubevirt_vmi_phase_count"))
Expect(phaseResultMetric.Value).To(BeEquivalentTo(1))
Expect(phaseResultMetric.Labels).To(HaveLen(7))
Expect(phaseResultMetric.Labels).To(HaveLen(11))
Expect(phaseResultMetric.Labels[6]).To(Equal(expected))
},
Entry("with no preference expect <none>", k6tv1.PreferenceAnnotation, "", "<none>"),
Expand Down
1 change: 1 addition & 0 deletions tests/monitoring/BUILD.bazel
View file
Expand Up @@ -39,6 +39,7 @@ go_library(
"//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
"//vendor/github.com/onsi/ginkgo/v2:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
"//vendor/github.com/onsi/gomega/types:go_default_library",
"//vendor/github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
Expand Down
42 changes: 42 additions & 0 deletions tests/monitoring/vm_monitoring.go
View file
Expand Up @@ -28,6 +28,7 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -246,6 +247,20 @@ var _ = Describe("[Serial][sig-monitoring]VM Monitoring", Serial, decorators.Sig
})
})

Context("VM metrics that are based on the guest agent", func() {
It("should have kubevirt_vmi_phase_count correctly configured with guest OS labels", func() {
agentVMI := createAgentVMI()
labels := map[string]string{
"guest_os_kernel_release": agentVMI.Status.GuestOSInfo.KernelRelease,
"guest_os_machine": agentVMI.Status.GuestOSInfo.Machine,
"guest_os_name": agentVMI.Status.GuestOSInfo.Name,
"guest_os_version_id": agentVMI.Status.GuestOSInfo.VersionID,
}

libmonitoring.WaitForMetricValueWithLabels(virtClient, "kubevirt_vmi_phase_count", 1, labels, 1)
})
})

Context("VM alerts", func() {
var scales *libmonitoring.Scaling

Expand Down Expand Up @@ -304,3 +319,30 @@ var _ = Describe("[Serial][sig-monitoring]VM Monitoring", Serial, decorators.Sig
})
})
})

func createAgentVMI() *v1.VirtualMachineInstance {
virtClient := kubevirt.Client()
agentVMI := libvmifact.NewFedora(libnet.WithMasqueradeNetworking()...)

By("Starting a VirtualMachineInstance")
agentVMI, err := virtClient.VirtualMachineInstance(testsuite.GetTestNamespace(agentVMI)).Create(context.Background(), agentVMI, metav1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())
libwait.WaitForSuccessfulVMIStart(agentVMI)

getOptions := metav1.GetOptions{}
var VMI *v1.VirtualMachineInstance

By("VMI has the guest agent connected condition")
Eventually(func() []v1.VirtualMachineInstanceCondition {
VMI, err = virtClient.VirtualMachineInstance(testsuite.GetTestNamespace(agentVMI)).Get(context.Background(), agentVMI.Name, getOptions)
Expect(err).ToNot(HaveOccurred())
return VMI.Status.Conditions
}, 240*time.Second, 1*time.Second).Should(
ContainElement(
MatchFields(
IgnoreExtras,
Fields{"Type": Equal(v1.VirtualMachineInstanceAgentConnected)})),
"Should have agent connected condition")

return VMI
}

0 comments on commit 7170053

Please sign in to comment.