Merge 659a937 into 254bc9b

sapcc · Mar 25, 2021 · ddf23eb · ddf23eb
2 parents 254bc9b + 659a937
commit ddf23eb
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 25 deletions.
diff --git a/BaseCollector.py b/BaseCollector.py
@@ -170,10 +170,17 @@ def generate_metrics(self, label_names: list) -> dict:
             logger.error(f'Cannot find {self.name} in collector_config')
         return metrics
 
-    def generate_metrics_renamed_by_api(self, metric_suffix, label_names):
-        gauge = GaugeMetricFamily(f'vrops_{self.vrops_entity_name}_{metric_suffix.lower()}',
-                                  'vrops-exporter', labels=label_names)
-        return gauge
+    def generate_metrics_enriched_by_api(self, no_match_in_config: list, label_names: list) -> dict:
+        gauges = dict()
+        for statkey in no_match_in_config:
+            new_metric_suffix = re.sub("[^0-9a-zA-Z]+", "_", statkey[0])
+            value = statkey[1]
+            labels = statkey[2]
+            if new_metric_suffix not in gauges:
+                gauges[new_metric_suffix] = GaugeMetricFamily(
+                    f'vrops_{self.vrops_entity_name}_{new_metric_suffix.lower()}', 'vrops-exporter', labels=label_names)
+            gauges[new_metric_suffix].add_metric(labels=labels, value=value)
+        return gauges
 
     def describe(self):
         collector_config = self.read_collector_config()

diff --git a/collectors/InventoryCollector.py b/collectors/InventoryCollector.py
@@ -13,6 +13,9 @@ def __init__(self):
         self.name = self.__class__.__name__
         self.wait_for_inventory_data()
 
+    def describe(self):
+        self.collect()
+
     def collect(self):
         logger.info(f'{self.name} starts with collecting the metrics')
 

diff --git a/collectors/StatsCollector.py b/collectors/StatsCollector.py
@@ -37,30 +37,29 @@ def collect(self):
             return
 
         values_received = set()
+        no_match_in_config = []
 
         for resource in values:
             resource_id = resource.get('resourceId')
             labels = self.set_labels(resource_id, project_ids)
 
             for value_entry in resource.get('stat-list', {}).get('stat', []):
                 statkey = value_entry.get('statKey', {}).get('key')
-                values_received.add(statkey)
+                # Normalisation of keys retrieved from API (e.g. cpu:102|usage_average -> cpu|usage_average)
+                norm_statkey = re.sub("[^a-zA-Z|_ ]+", "", statkey)
+                values_received.add(norm_statkey)
 
-                metric_data = value_entry.get('data', [None])[0]
-                if statkey in metrics and metric_data is not None:
-                    metrics[statkey]['gauge'].add_metric(labels=labels, value=metric_data)
+                metric_data = value_entry.get('data', [0])[0]
+                if norm_statkey in metrics:
+                    metrics[norm_statkey]['gauge'].add_metric(labels=labels, value=metric_data)
                 else:
-                    # no match in config, bring into the right format and yield
-                    new_metric_suffix = re.sub('[^a-zA-Z/s0-9\n.]', '_', statkey)
-                    values_received.add(new_metric_suffix)
-                    metrics[new_metric_suffix] = {}
-                    metrics[new_metric_suffix]['gauge'] = self.generate_metrics_renamed_by_api(new_metric_suffix,
-                                                                                               self.label_names)
-                    metrics[new_metric_suffix]['gauge'].add_metric(labels=labels, value=metric_data)
+                    no_match_in_config.append([statkey, metric_data, labels])
 
-        metrics_without_values = [m for m in metrics if m not in values_received]
-        if metrics_without_values:
-            logger.warning(f'No values for keys in {self.name}: {metrics_without_values} | Check renaming!')
+        # no match in config, bring into the right format
+        created_metrics = self.generate_metrics_enriched_by_api(no_match_in_config, label_names=self.label_names)
 
         for metric in metrics:
             yield metrics[metric]['gauge']
+        for metric in created_metrics:
+            logger.info(f'Created metrics enriched by API in {self.name}: {created_metrics[metric].name}')
+            yield created_metrics[metric]
diff --git a/tests/collector_config.yaml b/tests/collector_config.yaml
@@ -115,7 +115,7 @@ HostSystemStatsCollector:
   - metric_suffix: "network_packets_dropped_tx_number"
     key: "net|droppedTx_summation"
   - metric_suffix: "network_packets_dropped_percentage"
-    key: "net:Aggregate of all instances|droppedPct"
+    key: "net|droppedPct"
   - metric_suffix: "system_uptime_seconds"
     key: "sys|uptime_latest"
   - metric_suffix: "memory_swap_in_rate_kbps"
@@ -272,9 +272,9 @@ VMStatsDefaultCollector:
     key: "summary|number_datastore"
   - metric_suffix: "datastore_outstanding_io_requests"
     key: "datastore|demand_oio"
-  - metric_suffix: "database_disk_usage_gigabytes"
+  - metric_suffix: "guestfilesystem_storage_db_usage"
     key: "guestfilesystem:/storage/db|usage"
-  - metric_suffix: "database_disk_capacity_gigabytes"
+  - metric_suffix: "guestfilesystem_storage_db_capacity"
     key: "guestfilesystem:/storage/db|capacity"
-  - metric_suffix: "database_disk_usage_percent"
+  - metric_suffix: "guestfilesystem_storage_db_percentage"
     key: "guestfilesystem:/storage/db|percentage"
diff --git a/tests/metrics.yaml b/tests/metrics.yaml
@@ -154,10 +154,11 @@ VMStatsDefaultCollector:
   - 'vrops_virtualmachine_datastore_outstanding_io_requests{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
   - 'vrops_virtualmachine_diskspace_virtual_machine_used_gigabytes{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
   - 'vrops_virtualmachine_diskspace_gigabytes{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
-  - 'vrops_virtualmachine_database_disk_usage_percent{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
-  - 'vrops_virtualmachine_database_disk_usage_gigabytes{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
   - 'vrops_virtualmachine_disk_usage_average_kilobytes_per_second{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
-  - 'vrops_virtualmachine_database_disk_capacity_gigabytes{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
+  - 'vrops_virtualmachine_guestfilesystem_storage_db_capacity{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
+  - 'vrops_virtualmachine_guestfilesystem_storage_db_percentage{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
+  - 'vrops_virtualmachine_guestfilesystem_storage_db_usage{datacenter="datacenter3",hostsystem="hostsystem3",project="0815",vccluster="cluster3",vcenter="vcenter1",virtualmachine="vm1"}'
+
 
 VMPropertiesCollector:
   - 'vrops_api_response{class="vmpropertiescollector",target="testhost.test"}'