feat: initial draft of custom metric tool and its systemd timer (#1963)

To support the performance monitoring on mainnet, add a tool where
custom metrics can be calculated and exported to prometheus's
`node_exporter` through the `textfile` collector.

Currently, the total number of TLB shootdowns across all CPUs will be
exposed as `sum_tlb_shootdowns`, collected once per minute, as the
latest `node_exporter` does not allow filtering of data of its built-in
`interrupts` collector that could otherwise do it for us (until
prometheus/node_exporter#3028 is included in the
release branches) and will add many metrics with high cardinality
otherwise.

NODE-1445

Author: nabdullindfinity

Cargo.lock

@@ -126,6 +126,7 @@ members = [
     "rs/ic_os/build_tools/diroid",
     "rs/ic_os/config",
     "rs/ic_os/fstrim_tool",
+    "rs/ic_os/metrics_tool",
     "rs/ic_os/os_tools/guestos_tool",
     "rs/ic_os/os_tools/hostos_tool",
     "rs/ic_os/build_tools/inject_files",

Cargo.toml

@@ -85,6 +85,8 @@ component_files = {
     Label("monitoring/journald.conf"): "/etc/systemd/journald.conf",
     Label("monitoring/nft-exporter/nft-exporter.service"): "/etc/systemd/system/nft-exporter.service",
     Label("monitoring/nft-exporter/nft-exporter.timer"): "/etc/systemd/system/nft-exporter.timer",
+    Label("monitoring/custom-metrics/metrics_tool.service"): "/etc/systemd/system/metrics_tool.service",
+    Label("monitoring/custom-metrics/metrics_tool.timer"): "/etc/systemd/system/metrics_tool.timer",
 
     # networking
     Label("networking/generate-network-config/guestos/generate-network-config.service"): "/etc/systemd/system/generate-network-config.service",

ic-os/components/guestos.bzl

@@ -0,0 +1,33 @@
+[Unit]
+Description=Report custom metrics once per minute
+
+[Service]
+Type=oneshot
+ExecStart=/opt/ic/bin/metrics_tool --metrics /run/node_exporter/collector_textfile/custom_metrics.prom
+DeviceAllow=/dev/vda
+IPAddressDeny=any
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+NoNewPrivileges=yes
+PrivateDevices=no
+PrivateNetwork=yes
+PrivateTmp=yes
+PrivateUsers=no
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectSystem=strict
+ReadOnlyPaths=/proc/interrupts
+ReadWritePaths=/run/node_exporter/collector_textfile
+RestrictAddressFamilies=AF_UNIX
+RestrictAddressFamilies=~AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallErrorNumber=EPERM
+SystemCallFilter=@system-service
+UMask=022

ic-os/components/monitoring/custom-metrics/metrics_tool.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Collect custom metrics every minute
+
+[Timer]
+OnBootSec=60s
+OnUnitActiveSec=60s
+Unit=metrics_tool.service
+
+[Install]
+WantedBy=timers.target

ic-os/components/monitoring/custom-metrics/metrics_tool.timer

@@ -50,6 +50,7 @@ def image_deps(mode, malicious = False):
             "//cpp:infogetty": "/opt/ic/bin/infogetty:0755",  # Terminal manager that replaces the login shell.
             "//cpp:prestorecon": "/opt/ic/bin/prestorecon:0755",  # Parallel restorecon replacement for filesystem relabeling.
             "//rs/ic_os/release:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755",  # Proxies, filters, and serves public node metrics.
+            "//rs/ic_os/release:metrics_tool": "/opt/ic/bin/metrics_tool:0755",  # Collects and reports custom metrics.
 
             # additional libraries to install
             "//rs/ic_os/release:nss_icos": "/usr/lib/x86_64-linux-gnu/libnss_icos.so.2:0644",  # Allows referring to the guest IPv6 by name guestos from host, and host as hostos from guest.

ic-os/guestos/defs.bzl

@@ -0,0 +1,54 @@
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test", "rust_test_suite")
+
+package(default_visibility = ["//rs:ic-os-pkg"])
+
+DEPENDENCIES = [
+    # Keep sorted.
+    "//rs/sys",
+    "@crate_index//:anyhow",
+    "@crate_index//:clap",
+]
+
+DEV_DEPENDENCIES = [
+    # Keep sorted.
+]
+
+MACRO_DEPENDENCIES = []
+
+ALIASES = {}
+
+rust_library(
+    name = "metrics_tool",
+    srcs = glob(
+        ["src/**/*.rs"],
+        exclude = ["src/main.rs"],
+    ),
+    aliases = ALIASES,
+    crate_name = "ic_metrics_tool",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    visibility = ["//rs:system-tests-pkg"],
+    deps = DEPENDENCIES,
+)
+
+rust_binary(
+    name = "metrics_tool_bin",
+    srcs = ["src/main.rs"],
+    aliases = ALIASES,
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    deps = DEPENDENCIES + [":metrics_tool"],
+)
+
+rust_test(
+    name = "metrics_tool_test",
+    crate = ":metrics_tool",
+    deps = DEPENDENCIES + DEV_DEPENDENCIES,
+)
+
+rust_test_suite(
+    name = "metrics_tool_integration",
+    srcs = glob(["tests/**/*.rs"]),
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+    deps = [":metrics_tool_bin"] + DEPENDENCIES + DEV_DEPENDENCIES,
+)

rs/ic_os/metrics_tool/BUILD.bazel

@@ -0,0 +1,12 @@
+[package]
+name = "ic-metrics-tool"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "metrics_tool"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }

rs/ic_os/metrics_tool/Cargo.toml

@@ -0,0 +1,187 @@
+// TODO: refactor/merge this with fstrim_tool and guestos_tool metrics functionality
+use std::fs::File;
+use std::io::{self, Write};
+use std::path::Path;
+
+// TODO: everything is floating point for now
+pub struct Metric {
+    name: String,
+    value: f64,
+    annotation: String,
+    labels: Vec<(String, String)>,
+}
+
+impl Metric {
+    pub fn new(name: &str, value: f64) -> Self {
+        Self {
+            name: name.to_string(),
+            value,
+            annotation: "Custom metric".to_string(),
+            labels: Vec::new(),
+        }
+    }
+    pub fn with_annotation(name: &str, value: f64, annotation: &str) -> Self {
+        Self {
+            name: name.to_string(),
+            value,
+            annotation: annotation.to_string(),
+            labels: Vec::new(),
+        }
+    }
+
+    pub fn add_annotation(mut self, annotation: &str) -> Self {
+        self.annotation = annotation.to_string();
+        self
+    }
+
+    pub fn add_label(mut self, key: &str, value: &str) -> Self {
+        self.labels.push((key.to_string(), value.to_string()));
+        self
+    }
+
+    // TODO: formatting of floats
+    // Convert to prometheus exposition format
+    pub fn to_prom_string(&self) -> String {
+        let labels_str = if self.labels.is_empty() {
+            String::new()
+        } else {
+            let labels: Vec<String> = self
+                .labels
+                .iter()
+                .map(|(k, v)| format!("{}=\"{}\"", k, v))
+                .collect();
+            format!("{{{}}}", labels.join(","))
+        };
+        format!(
+            "# HELP {} {}\n\
+             # TYPE {} counter\n\
+             {}{} {}",
+            self.name, self.annotation, self.name, self.name, labels_str, self.value
+        )
+    }
+}
+
+pub struct MetricsWriter {
+    file_path: String,
+}
+
+impl MetricsWriter {
+    pub fn new(file_path: &str) -> Self {
+        Self {
+            file_path: file_path.to_string(),
+        }
+    }
+
+    pub fn write_metrics(&self, metrics: &[Metric]) -> io::Result<()> {
+        let path = Path::new(&self.file_path);
+        let mut file = File::create(path)?;
+        for metric in metrics {
+            writeln!(file, "{}", metric.to_prom_string())?;
+        }
+        Ok(())
+    }
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_metric_to_string() {
+        let metric = Metric::new("test_metric", 123.45)
+            .add_label("label1", "value1")
+            .add_label("label2", "value2");
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP test_metric Custom metric\n\
+             # TYPE test_metric counter\n\
+             test_metric{label1=\"value1\",label2=\"value2\"} 123.45"
+        );
+    }
+
+    #[test]
+    fn test_write_metrics() {
+        let metrics = vec![
+            Metric::new("metric1", 1.0),
+            Metric::new("metric2", 2.0).add_label("label", "value"),
+        ];
+        let writer = MetricsWriter::new("/tmp/test_metrics.prom");
+        writer.write_metrics(&metrics).unwrap();
+        let content = std::fs::read_to_string("/tmp/test_metrics.prom").unwrap();
+        assert!(content.contains(
+            "# HELP metric1 Custom metric\n\
+             # TYPE metric1 counter\n\
+             metric1 1"
+        ));
+        assert!(content.contains(
+            "# HELP metric2 Custom metric\n\
+             # TYPE metric2 counter\n\
+             metric2{label=\"value\"} 2"
+        ));
+    }
+
+    #[test]
+    fn test_metric_large_value() {
+        let metric = Metric::new("large_value_metric", 1.0e64);
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP large_value_metric Custom metric\n\
+             # TYPE large_value_metric counter\n\
+             large_value_metric 10000000000000000000000000000000000000000000000000000000000000000"
+        );
+    }
+
+    #[test]
+    fn test_metric_without_labels() {
+        let metric = Metric::new("no_label_metric", 42.0);
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP no_label_metric Custom metric\n\
+             # TYPE no_label_metric counter\n\
+             no_label_metric 42"
+        );
+    }
+
+    #[test]
+    fn test_metric_with_annotation() {
+        let metric = Metric::with_annotation("annotated_metric", 99.9, "This is a test metric");
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP annotated_metric This is a test metric\n\
+             # TYPE annotated_metric counter\n\
+             annotated_metric 99.9"
+        );
+    }
+
+    #[test]
+    fn test_write_empty_metrics() {
+        let metrics: Vec<Metric> = Vec::new();
+        let writer = MetricsWriter::new("/tmp/test_empty_metrics.prom");
+        writer.write_metrics(&metrics).unwrap();
+        let content = std::fs::read_to_string("/tmp/test_empty_metrics.prom").unwrap();
+        assert!(content.is_empty());
+    }
+
+    #[test]
+    fn test_metric_with_multiple_labels() {
+        let metric = Metric::new("multi_label_metric", 10.0)
+            .add_label("foo", "bar")
+            .add_label("version", "1.0.0");
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP multi_label_metric Custom metric\n\
+             # TYPE multi_label_metric counter\n\
+             multi_label_metric{foo=\"bar\",version=\"1.0.0\"} 10"
+        );
+    }
+
+    #[test]
+    fn test_metric_with_empty_annotation() {
+        let metric = Metric::with_annotation("empty_annotation_metric", 5.5, "");
+        assert_eq!(
+            metric.to_prom_string(),
+            "# HELP empty_annotation_metric \n\
+             # TYPE empty_annotation_metric counter\n\
+             empty_annotation_metric 5.5"
+        );
+    }
+}