Handle exceptions from collectors (#3044)

* wrapping collectors with try blocks

* remove unused e

* items not enumerate

* extract task_name, not name

* started_at, not time_start

discovery-provider/src/queries/get_celery_tasks.py

@@ -1,6 +1,9 @@
+import logging
+
 from src.monitors import monitor_names, monitors
 from src.utils.prometheus_metric import PrometheusMetric, PrometheusType
 
+logger = logging.getLogger(__name__)
 MONITORS = monitors.MONITORS
 
 
@@ -27,7 +30,12 @@ def celery_tasks_prometheus_exporter():
     )
 
     for task in tasks:
-        metric.save_time({"task_name": task["name"]}, start_time=task["time_start"])
+        try:
+            metric.save_time(
+                {"task_name": task["task_name"]}, start_time=task["started_at"]
+            )
+        except:
+            logger.exception(f"Processing failed for task: {task}")
 
 
 PrometheusMetric.register_collector(

discovery-provider/src/utils/prometheus_metric.py

@@ -1,8 +1,11 @@
+import logging
 from time import time
 from typing import Callable, Dict
 
 from prometheus_client import Gauge, Histogram
 
+logger = logging.getLogger(__name__)
+
 
 class PrometheusType:
     HISTOGRAM = "histogram"
@@ -73,5 +76,8 @@ def register_collector(cls, name, collector_func):
 
     @classmethod
     def populate_collectors(cls):
-        for collector in cls.registered_collectors.values():
-            collector()
+        for name, collector in cls.registered_collectors.items():
+            try:
+                collector()
+            except:
+                logger.exception(f"Failure to collect '{name}'")