Feature: add metrics in metasrv

common/metrics/src/lib.rs

@@ -20,8 +20,15 @@ pub use dump::HistogramCount;
 pub use dump::MetricSample;
 pub use dump::MetricValue;
 pub use dump::SummaryCount;
+pub use metrics::Counter;
+pub use metrics::Gauge;
+pub use metrics::Histogram;
 pub use metrics_exporter_prometheus::PrometheusHandle;
 pub use recorder::init_default_metrics_recorder;
 pub use recorder::label_counter;
 pub use recorder::label_counter_with_val;
+pub use recorder::register_counter;
+pub use recorder::register_gauge;
+pub use recorder::register_histogram;
 pub use recorder::try_handle;
+pub use recorder::MetricOption;

common/metrics/src/recorder.rs

@@ -18,6 +18,12 @@ use std::sync::Once;
 use common_infallible::RwLock;
 use common_tracing::tracing;
 use metrics::counter;
+use metrics::register_counter;
+use metrics::register_gauge;
+use metrics::register_histogram;
+use metrics::Counter;
+use metrics::Gauge;
+use metrics::Histogram;
 use metrics_exporter_prometheus::PrometheusBuilder;
 use metrics_exporter_prometheus::PrometheusHandle;
 use once_cell::sync::Lazy;
@@ -28,6 +34,28 @@ static PROMETHEUS_HANDLE: Lazy<Arc<RwLock<Option<PrometheusHandle>>>> =
 pub const LABEL_KEY_TENANT: &str = "tenant";
 pub const LABEL_KEY_CLUSTER: &str = "cluster_name";
 
+pub const LABEL_NAMESPACE: &str = "namespace";
+pub const LABEL_SUBSYSTEM: &str = "subsystem";
+pub const LABEL_HELP: &str = "help";
+
+pub struct MetricOption {
+    name: String,
+    namespace: String,
+    subsystem: String,
+    help: String,
+}
+
+impl MetricOption {
+    pub fn new(name: String, namespace: String, subsystem: String, help: String) -> MetricOption {
+        MetricOption {
+            name,
+            namespace,
+            subsystem,
+            help,
+        }
+    }
+}
+
 #[inline]
 pub fn label_counter(name: &'static str, tenant_id: &str, cluster_id: &str) {
     label_counter_with_val(name, 1, tenant_id, cluster_id)
@@ -62,3 +90,30 @@ fn init_prometheus_recorder() {
 pub fn try_handle() -> Option<PrometheusHandle> {
     PROMETHEUS_HANDLE.as_ref().read().clone()
 }
+
+pub fn register_counter(opt: MetricOption) -> Counter {
+    let labels = [
+        (LABEL_NAMESPACE, opt.namespace.to_string()),
+        (LABEL_SUBSYSTEM, opt.subsystem.to_string()),
+        (LABEL_HELP, opt.help.to_string()),
+    ];
+    register_counter!(opt.name, &labels)
+}
+
+pub fn register_gauge(opt: MetricOption) -> Gauge {
+    let labels = [
+        (LABEL_NAMESPACE, opt.namespace.to_string()),
+        (LABEL_SUBSYSTEM, opt.subsystem.to_string()),
+        (LABEL_HELP, opt.help.to_string()),
+    ];
+    register_gauge!(opt.name, &labels)
+}
+
+pub fn register_histogram(opt: MetricOption) -> Histogram {
+    let labels = [
+        (LABEL_NAMESPACE, opt.namespace.to_string()),
+        (LABEL_SUBSYSTEM, opt.subsystem.to_string()),
+        (LABEL_HELP, opt.help.to_string()),
+    ];
+    register_histogram!(opt.name, &labels)
+}

metasrv/Cargo.toml

@@ -29,6 +29,7 @@ common-base = { path = "../common/base" }
 common-containers = { path = "../common/containers" }
 common-exception = { path = "../common/exception" }
 common-grpc = { path = "../common/grpc" }
+common-infallible = { path = "../common/infallible" }
 common-macros = { path = "../common/macros" }
 common-meta-api = { path = "../common/meta/api" }
 common-meta-grpc = { path = "../common/meta/grpc" }
@@ -50,6 +51,7 @@ clap = { version = "3.1.8", features = ["derive", "env"] }
 futures = "0.3.21"
 metrics = "0.18.1"
 num = "0.4.0"
+once_cell = "1.10.0"
 poem = { version = "=1.3.16", features = ["rustls"] }
 prost = "=0.9.0"
 serde = { version = "1.0.136", features = ["derive"] }

metasrv/src/bin/metasrv.rs

@@ -26,6 +26,7 @@ use databend_meta::api::GrpcServer;
 use databend_meta::api::HttpService;
 use databend_meta::configs::Config;
 use databend_meta::meta_service::MetaNode;
+use databend_meta::metrics::init_meta_metrics_recorder;
 use databend_meta::metrics::MetricService;
 
 #[databend_main]
@@ -42,6 +43,7 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
 
     init_sled_db(conf.raft_config.raft_dir.clone());
     init_default_metrics_recorder();
+    init_meta_metrics_recorder();
 
     tracing::info!(
         "Starting MetaNode single: {} with config: {:?}",

metasrv/src/meta_service/raftmeta.rs

@@ -65,6 +65,7 @@ use crate::meta_service::meta_leader::MetaLeader;
 use crate::meta_service::ForwardRequestBody;
 use crate::meta_service::JoinRequest;
 use crate::meta_service::RaftServiceImpl;
+use crate::metrics::set_meta_metrics_has_leader;
 use crate::network::Network;
 use crate::store::MetaRaftStore;
 use crate::watcher::WatcherManager;
@@ -400,6 +401,9 @@ impl MetaNode {
                                         );
                                     }
                                 }
+                                set_meta_metrics_has_leader(true);
+                            } else {
+                                set_meta_metrics_has_leader(false);
                             }
                         } else {
                             // shutting down

metasrv/src/metrics/meta_metrics.rs

@@ -0,0 +1,78 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Once;
+
+use common_metrics::Gauge;
+use common_metrics::MetricOption;
+use once_cell::sync::OnceCell;
+
+pub const META_NAMESPACE: &str = "metasrv";
+pub const SERVER_SUBSYSTEM: &str = "server";
+
+pub struct MetaMetrics {
+    has_leader: Option<Gauge>,
+}
+
+static INSTANCE: OnceCell<MetaMetrics> = OnceCell::new();
+
+impl Default for MetaMetrics {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl MetaMetrics {
+    pub fn new() -> MetaMetrics {
+        MetaMetrics { has_leader: None }
+    }
+
+    pub fn instance() -> &'static MetaMetrics {
+        INSTANCE.get().expect("metametrics is not initialized")
+    }
+
+    /// Init all meta metrics.
+    pub fn register(self: &mut MetaMetrics) {
+        // add has_leader metric
+        let has_leader = MetricOption::new(
+            "has_leader".to_string(),
+            META_NAMESPACE.to_string(),
+            SERVER_SUBSYSTEM.to_string(),
+            "Whether or not a leader exists.".to_string(),
+        );
+        self.has_leader = Some(common_metrics::register_gauge(has_leader));
+    }
+
+    pub fn has_leader(&self, has_leader: bool) {
+        let a = self.has_leader.as_ref().unwrap();
+        a.set(if has_leader { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn init_meta_metrics_recorder() {
+    static START: Once = Once::new();
+    START.call_once(init_meta_recorder)
+}
+
+/// Init meta metrics recorder.
+fn init_meta_recorder() {
+    let mut meta_metrics = MetaMetrics::new();
+    meta_metrics.register();
+
+    let _ = INSTANCE.set(meta_metrics);
+}
+
+pub fn set_meta_metrics_has_leader(has_leader: bool) {
+    MetaMetrics::instance().has_leader(has_leader);
+}

metasrv/src/metrics/metric_service.rs

@@ -49,6 +49,7 @@ impl MetricService {
         let prometheus_handle = common_metrics::try_handle().ok_or_else(|| {
             ErrorCode::InitPrometheusFailure("Prometheus recorder has not been initialized yet.")
         })?;
+
         let app = poem::Route::new()
             .at("/metrics", poem::get(metric_handler))
             .data(prometheus_handle);

metasrv/src/metrics/mod.rs

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod meta_metrics;
 mod metric_service;
 
+pub use meta_metrics::init_meta_metrics_recorder;
+pub use meta_metrics::set_meta_metrics_has_leader;
+pub use meta_metrics::MetaMetrics;
 pub use metric_service::MetricService;