test(metrics): add test to catch breaking change in vhost-user metrics

Add schema to validate breaking change in vhost-user metrics.
Re-organize metrics validation function to accommodate vhost-user
which has per device metrics but does not have aggregate metrics.
Use existing vhost-user tests to check that there is no breaking
change in vhost-user-block device metrics.

Signed-off-by: Sudan Landge <sudanl@amazon.com>

tests/host_tools/metrics.py

@@ -205,14 +205,14 @@ def format_with_reduced_unit(value, unit):
 
     return f"{reduced_value:.2f}{formatted_unit}"
 
+
 def validate_fc_metrics(metrics):
     """
     This functions makes sure that all components
-    of FirecrackerMetrics struct are present.
+    of firecracker_metrics struct are present.
     """
 
-
-    FirecrackerMetrics = {
+    firecracker_metrics = {
         "api_server": [
             "process_startup_time_us",
             "process_startup_time_cpu_us",
@@ -431,19 +431,31 @@ def validate_fc_metrics(metrics):
     assert abs(utc_timestamp_ms - metrics["utc_timestamp_ms"]) < 1000
 
     if platform.machine() == "aarch64":
-        FirecrackerMetrics["rtc"] = [
+        firecracker_metrics["rtc"] = [
             "error_count",
             "missed_read_count",
             "missed_write_count",
         ]
 
+    # add vhost-user metrics to the schema if applicable
+    vhost_user_devices = []
+    for metrics_name in metrics.keys():
+        if metrics_name.startswith("vhost_user_"):
+            firecracker_metrics[metrics_name] = [
+                "activate_fails",
+                "cfg_fails",
+                "init_time_us",
+                "activate_time_us",
+            ]
+            vhost_user_devices.append(metrics_name)
+
     firecracker_metrics_schema = {
         "type": "object",
         "properties": {},
         "required": [],
     }
 
-    for metrics_name, metrics_fields in FirecrackerMetrics.items():
+    for metrics_name, metrics_fields in firecracker_metrics.items():
         metrics_schema = {
             "type": "object",
             "required": metrics_fields,
@@ -482,6 +494,18 @@ def validate_missing_metrics(metrics):
                 else:
                     raise error
             metrics["rtc"]["error_count"] = temp_pop_metrics
+
+        for vhost_user_dev in vhost_user_devices:
+            temp_pop_metrics = metrics[vhost_user_dev].pop("activate_time_us")
+            try:
+                jsonschema.validate(instance=metrics, schema=firecracker_metrics_schema)
+            except jsonschema.exceptions.ValidationError as error:
+                if error.message.strip() == "'activate_time_us' is a required property":
+                    pass
+                else:
+                    raise error
+            metrics[vhost_user_dev]["activate_time_us"] = temp_pop_metrics
+
     validate_missing_metrics(metrics)
 
 
@@ -491,37 +515,42 @@ class FcDeviceMetrics:
     aggregation of metrics
     """
 
-    def __init__(self, name, num_dev):
+    def __init__(self, name, num_dev, aggr_supported=True):
         self.dev_name = name
         self.num_dev = num_dev
+        self.aggr_supported = aggr_supported
 
     def validate(self, microvm):
         """
         validate breaking change of device metrics
         """
         fc_metrics = microvm.flush_metrics()
 
-        # make sure all items of FirecrackerMetrics are as expected
+        # make sure all items of firecracker_metrics are as expected
         validate_fc_metrics(fc_metrics)
 
         # make sure "{self.name}" is aggregate of "{self.name}_*"
         # and that there are only {num_dev} entries of "{self.name}_*"
-        self.validate_aggregation(fc_metrics)
+        self.validate_per_device_metrics(fc_metrics)
 
-    def validate_aggregation(self, fc_metrics):
+    def validate_per_device_metrics(self, fc_metrics):
         """
         validate aggregation of device metrics
         """
-        metrics_aggregate = fc_metrics[self.dev_name]
         metrics_calculated = {}
         actual_num_devices = 0
         for component_metric_names, component_metric_values in fc_metrics.items():
-            if f"{self.dev_name}_" in component_metric_names:
+            if (
+                f"{self.dev_name}_" in component_metric_names
+                and component_metric_names.startswith(self.dev_name)
+            ):
                 actual_num_devices += 1
                 for metrics_name, metric_value in component_metric_values.items():
                     if metrics_name not in metrics_calculated:
                         metrics_calculated[metrics_name] = 0
                     metrics_calculated[metrics_name] += metric_value
 
-        assert metrics_aggregate == metrics_calculated
         assert self.num_dev == actual_num_devices
+        if self.aggr_supported:
+            metrics_aggregate = fc_metrics[self.dev_name]
+            assert metrics_aggregate == metrics_calculated

tests/integration_tests/functional/test_drive_vhost_user.py

@@ -10,6 +10,7 @@
 from framework import utils
 from framework.defs import LOCAL_BUILD_PATH
 from framework.utils_drive import partuuid_and_disk_path, spawn_vhost_user_backend
+from host_tools.metrics import FcDeviceMetrics
 
 
 @pytest.fixture
@@ -69,11 +70,15 @@ def test_vhost_user_block(microvm_factory, guest_kernel, rootfs_ubuntu_22):
     vm.basic_config(add_root_device=False)
     vm.add_vhost_user_drive("rootfs", vhost_user_socket, is_root_device=True)
     vm.add_net_iface()
+    vhost_user_block_metrics = FcDeviceMetrics(
+        "vhost_user_block", 1, aggr_supported=False
+    )
     vm.start()
 
     # Attempt to connect to the VM.
     # Verify if guest can run commands.
     exit_code, _, _ = vm.ssh.run("ls")
+
     assert exit_code == 0
 
     # Now check that vhost-user-block with rw is last.
@@ -83,6 +88,7 @@ def test_vhost_user_block(microvm_factory, guest_kernel, rootfs_ubuntu_22):
         "1-6": "/dev/vda",
     }
     _check_drives(vm, assert_dict, assert_dict.keys())
+    vhost_user_block_metrics.validate(vm)
 
 
 def test_vhost_user_block_read_write(microvm_factory, guest_kernel, rootfs_ubuntu_22):
@@ -206,6 +212,10 @@ def test_device_ordering(microvm_factory, guest_kernel, rootfs_ubuntu_22):
     # Adding forth block device.
     vm.add_vhost_user_drive("dummy_rootfs", vhost_user_socket_2)
 
+    block_metrics = FcDeviceMetrics("block", 2, aggr_supported=True)
+    vhost_user_block_metrics = FcDeviceMetrics(
+        "vhost_user_block", 2, aggr_supported=False
+    )
     vm.start()
 
     rootfs_size = rootfs_ubuntu_22.stat().st_size
@@ -233,6 +243,8 @@ def test_device_ordering(microvm_factory, guest_kernel, rootfs_ubuntu_22):
         "4-6": "/dev/vdd",
     }
     _check_drives(vm, assert_dict, assert_dict.keys())
+    block_metrics.validate(vm)
+    vhost_user_block_metrics.validate(vm)
 
 
 def test_partuuid_boot(
@@ -316,6 +328,9 @@ def test_partuuid_update(microvm_factory, guest_kernel, rootfs_ubuntu_22):
     _backend = spawn_vhost_user_backend(vm, rootfs_path, vhost_user_socket_2, True)
     vm.add_vhost_user_drive("rootfs", vhost_user_socket_2, is_root_device=True)
 
+    vhost_user_block_metrics = FcDeviceMetrics(
+        "vhost_user_block", 1, aggr_supported=False
+    )
     vm.start()
 
     # Attempt to connect to the VM.
@@ -330,3 +345,4 @@ def test_partuuid_update(microvm_factory, guest_kernel, rootfs_ubuntu_22):
         "1-6": "/dev/vda",
     }
     _check_drives(vm, assert_dict, assert_dict.keys())
+    vhost_user_block_metrics.validate(vm)