mgr: prometheus: added osd commit/apply latency metrics (#22718).

Signed-off-by: Konstantin Shalygin <k0ste@k0ste.ru>

src/pybind/mgr/prometheus/module.py

@@ -56,6 +56,8 @@ def health_status_to_number(status):
 
 OSD_STATUS = ['weight', 'up', 'in']
 
+OSD_STATS = ['apply_latency_ms', 'commit_latency_ms']
+
 POOL_METADATA = ('pool_id', 'name')
 
 DISK_OCCUPATION = ('instance', 'device', 'ceph_daemon')
@@ -202,6 +204,15 @@ def _setup_static_metrics(self):
                 'OSD status {}'.format(state),
                 ('ceph_daemon',)
             )
+        for stat in OSD_STATS:
+            path = 'osd_{}'.format(stat)
+            self.log.debug("init: creating {}".format(path))
+            metrics[path] = Metric(
+                'gauge',
+                path,
+                'OSD stat {}'.format(stat),
+                ('ceph_daemon',)
+            )
         for state in PG_STATES:
             path = 'pg_{}'.format(state)
             self.log.debug("init: creating {}".format(path))
@@ -276,6 +287,16 @@ def get_pg_status(self):
                 except KeyError:
                     self.log.warn("skipping pg in unknown state {}".format(state))
 
+    def get_osd_stats(self):
+        osd_stats = self.get('osd_stats')
+        for osd in osd_stats['osd_stats']:
+            id_ = osd['osd']
+            for stat in OSD_STATS:
+                status = osd['perf_stat'][stat]
+                self.metrics['osd_{}'.format(stat)].set(
+                    status,
+                    ('osd.{}'.format(id_),))
+
     def get_metadata_and_osd_status(self):
         osd_map = self.get('osd_map')
         osd_devices = self.get('osd_map_crush')['devices']
@@ -325,6 +346,7 @@ def get_metadata_and_osd_status(self):
     def collect(self):
         self.get_health()
         self.get_df()
+        self.get_osd_stats()
         self.get_quorum_status()
         self.get_metadata_and_osd_status()
         self.get_pg_status()