Added a metric for number of indexes (#4711)

quickwit/quickwit-control-plane/src/control_plane.rs

@@ -61,7 +61,7 @@ use crate::debouncer::Debouncer;
 use crate::indexing_scheduler::{IndexingScheduler, IndexingSchedulerState};
 use crate::ingest::ingest_controller::IngestControllerStats;
 use crate::ingest::IngestController;
-use crate::model::{ControlPlaneModel, ControlPlaneModelMetrics};
+use crate::model::ControlPlaneModel;
 use crate::IndexerPool;
 
 /// Interval between two controls (or checks) of the desired plan VS running plan.
@@ -139,7 +139,8 @@ impl ControlPlane {
 pub struct ControlPlaneObservableState {
     pub indexing_scheduler: IndexingSchedulerState,
     pub ingest_controller: IngestControllerStats,
-    pub model_metrics: ControlPlaneModelMetrics,
+    pub num_indexes: usize,
+    pub num_sources: usize,
 }
 
 #[async_trait]
@@ -154,7 +155,8 @@ impl Actor for ControlPlane {
         ControlPlaneObservableState {
             indexing_scheduler: self.indexing_scheduler.observable_state(),
             ingest_controller: self.ingest_controller.stats,
-            model_metrics: self.model.observable_state(),
+            num_indexes: self.model.num_indexes(),
+            num_sources: self.model.num_sources(),
         }
     }
 
@@ -2003,8 +2005,8 @@ mod tests {
         assert!(matches!(error, ControlPlaneError::Unavailable { .. }));
 
         let control_plane_state = control_plane_mailbox.ask(Observe).await.unwrap();
-        assert_eq!(control_plane_state.model_metrics.num_indexes, 1);
-        assert_eq!(control_plane_state.model_metrics.num_sources, 1);
+        assert_eq!(control_plane_state.num_indexes, 1);
+        assert_eq!(control_plane_state.num_sources, 1);
 
         universe.assert_quit().await;
     }

quickwit/quickwit-control-plane/src/ingest/ingest_controller.rs

@@ -1175,7 +1175,7 @@ mod tests {
             GetOrCreateOpenShardsFailureReason::SourceNotFound
         );
 
-        assert_eq!(model.observable_state().num_shards, 3);
+        assert_eq!(model.num_shards(), 3);
     }
 
     #[tokio::test]

quickwit/quickwit-control-plane/src/metrics.rs

@@ -18,9 +18,12 @@
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 use once_cell::sync::Lazy;
-use quickwit_common::metrics::{new_counter, new_gauge_vec, IntCounter, IntGaugeVec};
+use quickwit_common::metrics::{
+    new_counter, new_gauge, new_gauge_vec, IntCounter, IntGauge, IntGaugeVec,
+};
 
 pub struct ControlPlaneMetrics {
+    pub indexes_total: IntGauge,
     pub restart_total: IntCounter,
     pub schedule_total: IntCounter,
     pub metastore_error_aborted: IntCounter,
@@ -31,6 +34,7 @@ pub struct ControlPlaneMetrics {
 impl Default for ControlPlaneMetrics {
     fn default() -> Self {
         ControlPlaneMetrics {
+            indexes_total: new_gauge("indexes_total", "Number of indexes.", "control_plane", &[]),
             restart_total: new_counter(
                 "restart_total",
                 "Number of control plane restart.",

quickwit/quickwit-control-plane/src/model/mod.rs

@@ -38,7 +38,6 @@ use quickwit_proto::metastore::{
     MetastoreError, MetastoreService, MetastoreServiceClient, SourceType,
 };
 use quickwit_proto::types::{IndexId, IndexUid, NodeId, ShardId, SourceId, SourceUid};
-use serde::Serialize;
 pub(super) use shard_table::{ScalingMode, ShardEntry, ShardStats, ShardTable};
 use tracing::{info, instrument, warn};
 
@@ -58,25 +57,23 @@ pub(crate) struct ControlPlaneModel {
     shard_table: ShardTable,
 }
 
-#[derive(Clone, Copy, Debug, Default, Serialize)]
-pub struct ControlPlaneModelMetrics {
-    pub num_indexes: usize,
-    pub num_sources: usize,
-    pub num_shards: usize,
-}
-
 impl ControlPlaneModel {
     /// Clears the entire state of the model.
     pub fn clear(&mut self) {
         *self = Default::default();
     }
 
-    pub fn observable_state(&self) -> ControlPlaneModelMetrics {
-        ControlPlaneModelMetrics {
-            num_indexes: self.index_table.len(),
-            num_sources: self.shard_table.num_sources(),
-            num_shards: self.shard_table.num_shards(),
-        }
+    pub fn num_indexes(&self) -> usize {
+        self.index_table.len()
+    }
+
+    pub fn num_sources(&self) -> usize {
+        self.shard_table.num_sources()
+    }
+
+    #[cfg(test)]
+    pub fn num_shards(&self) -> usize {
+        self.shard_table.num_shards()
     }
 
     #[instrument(skip_all)]
@@ -152,6 +149,12 @@ impl ControlPlaneModel {
         self.index_uid_table.get(index_id).cloned()
     }
 
+    fn update_metrics(&self) {
+        crate::metrics::CONTROL_PLANE_METRICS
+            .indexes_total
+            .set(self.index_table.len() as i64);
+    }
+
     pub(crate) fn source_configs(&self) -> impl Iterator<Item = (SourceUid, &SourceConfig)> + '_ {
         self.index_table.values().flat_map(|index_metadata| {
             index_metadata
@@ -181,12 +184,14 @@ impl ControlPlaneModel {
             }
         }
         self.index_table.insert(index_uid, index_metadata);
+        self.update_metrics();
     }
 
     pub(crate) fn delete_index(&mut self, index_uid: &IndexUid) {
         self.index_table.remove(index_uid);
         self.index_uid_table.remove(&index_uid.index_id);
         self.shard_table.delete_index(&index_uid.index_id);
+        self.update_metrics();
     }
 
     /// Adds a source to a given index. Returns an error if the source already

quickwit/quickwit-control-plane/src/model/shard_table.rs

@@ -209,6 +209,7 @@ impl ShardTable {
         self.table_entries.len()
     }
 
+    #[cfg(test)]
     pub fn num_shards(&self) -> usize {
         self.table_entries
             .values()