From 25dbdaa4fd20ddda33f528f58f27227a30d0b4c9 Mon Sep 17 00:00:00 2001
From: Luca Cominardi <luca.cominardi@datadoghq.com>
Date: Mon, 3 Aug 2026 09:58:54 +0200
Subject: [PATCH 1/5] Add global metrics labels env var

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/reference/metrics.md              | 10 +++++++
 quickwit/quickwit-metrics/src/inner.rs | 40 +++++++++++++++++++++++++-
 quickwit/quickwit-metrics/src/lib.rs   |  5 +++-
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md
index e75121f08f3..f740287c242 100644
--- a/docs/reference/metrics.md
+++ b/docs/reference/metrics.md
@@ -91,3 +91,13 @@ PostgreSQL-backed metastores also expose connection pool gauges:
 | `quickwit_storage` | `object_storage_puts_total` | Number of objects uploaded. May differ from object_storage_requests_parts due to multipart upload | `counter` |
 | `quickwit_storage` | `object_storage_puts_parts` | Number of object parts uploaded | `counter` |
 | `quickwit_storage` | `object_storage_download_num_bytes` | Amount of data downloaded from an object storage | `counter` |
+
+## Global labels
+
+You can attach the same labels to every Quickwit metric by setting the `QW_METRICS_LABELS` environment variable. Specify labels as a comma-separated list of `name=value` pairs:
+
+```bash
+QW_METRICS_LABELS="environment=production,region=us-east-1" quickwit run
+```
+
+The environment variable is read once, when the first metric is initialized. Set it before starting Quickwit; changing it while Quickwit is running has no effect.
diff --git a/quickwit/quickwit-metrics/src/inner.rs b/quickwit/quickwit-metrics/src/inner.rs
index 69f0858a971..93c5374dffc 100644
--- a/quickwit/quickwit-metrics/src/inner.rs
+++ b/quickwit/quickwit-metrics/src/inner.rs
@@ -19,12 +19,40 @@
 //! items via `$crate::`. **Do not use directly.**
 
 use std::hash::{Hash, Hasher};
+use std::sync::LazyLock;
 
 #[doc(hidden)]
 pub use const_format::concatcp as __concatcp;
+use metrics::Label;
 use rustc_hash::FxHasher;
 
+static METRICS_LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
+        // quickwit-common defines common helpers for getting environment variables.
+        // However, we need to use the `std::env::var` function directly here because
+        // quickwit-common depends on quickwit-metrics and it would cause a circular dependency.
+
+        // The format of the environment variable is:
+        // QW_METRICS_LABELS="environment=test,region=us-east-1,foo=bar"
+        match std::env::var(super::METRICS_LABELS_ENV_VAR_NAME) {
+            Ok(labels) => {
+                let labels: Vec<Label> = labels
+                    .split(',')
+                    .flat_map(|label| label.split_once('='))
+                    .map(|(name, value)| Label::new(
+                        name.trim().to_string(), value.trim().to_string()
+                    ))
+                    .collect();
+                labels.into_boxed_slice()
+            },
+            Err(_) => Vec::new().into_boxed_slice(),
+        });
+
 // ─── Helper functions ───
+/// Returns the labels from the environment variable.
+#[doc(hidden)]
+pub fn __labels_env_var() -> &'static [Label] {
+    METRICS_LABELS_ENV_VAR.as_ref()
+}
 
 /// Counts the number of token-tree arguments at compile time.
 #[doc(hidden)]
@@ -94,7 +122,17 @@ macro_rules! __key_info_metadata {
         static LABELS: [$crate::__metrics::Label; $crate::__count!($($label)*)] = [
             $($crate::__metrics::Label::from_static_parts($label, $value)),*
         ];
-        static KEY: $crate::__metrics::Key = $crate::__metrics::Key::from_static_parts(KEY_NAME, &LABELS);
+        static KEY: std::sync::LazyLock<$crate::__metrics::Key> = std::sync::LazyLock::new(|| {
+            let labels_env_var = $crate::__labels_env_var();
+            if labels_env_var.is_empty() {
+                $crate::__metrics::Key::from_static_parts(KEY_NAME, &LABELS)
+            } else {
+                let mut labels = Vec::with_capacity(LABELS.len() + labels_env_var.len());
+                labels.extend(LABELS.iter().cloned());
+                labels.extend(labels_env_var.iter().cloned());
+                $crate::__metrics::Key::from_parts(KEY_NAME, labels)
+            }
+        });
     };
 }
 
diff --git a/quickwit/quickwit-metrics/src/lib.rs b/quickwit/quickwit-metrics/src/lib.rs
index bf1992ef2fd..27eff481320 100644
--- a/quickwit/quickwit-metrics/src/lib.rs
+++ b/quickwit/quickwit-metrics/src/lib.rs
@@ -268,6 +268,9 @@
 pub const SYSTEM: &str = "quickwit";
 pub const SEPARATOR: &str = "_";
 
+/// The name of the environment variable that contains the injected labels for all the metrics.
+pub(crate) const METRICS_LABELS_ENV_VAR_NAME: &str = "QW_METRICS_LABELS";
+
 // ─── Metric modules ───
 mod counter;
 mod gauge;
@@ -288,7 +291,7 @@ pub use gauge::__gauge_get_or_register;
 #[doc(hidden)]
 pub use histogram::__histogram_get_or_register;
 #[doc(hidden)]
-pub use inner::{__concatcp, __key_hash, __sep};
+pub use inner::{__concatcp, __key_hash, __labels_env_var, __sep};
 
 // Re-exports of `metrics` and `inventory` used inside macro expansions.
 #[doc(hidden)]

From 0df1708f24850b9d9561c686fca960409bf7d42a Mon Sep 17 00:00:00 2001
From: Luca Cominardi <luca.cominardi@datadoghq.com>
Date: Mon, 3 Aug 2026 10:00:19 +0200
Subject: [PATCH 2/5] Name metrics label separators

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 quickwit/quickwit-metrics/src/inner.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/quickwit/quickwit-metrics/src/inner.rs b/quickwit/quickwit-metrics/src/inner.rs
index 93c5374dffc..ff02174ad44 100644
--- a/quickwit/quickwit-metrics/src/inner.rs
+++ b/quickwit/quickwit-metrics/src/inner.rs
@@ -35,9 +35,12 @@ static METRICS_LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
         // QW_METRICS_LABELS="environment=test,region=us-east-1,foo=bar"
         match std::env::var(super::METRICS_LABELS_ENV_VAR_NAME) {
             Ok(labels) => {
+                const LABELS_SEPARATOR: char = ',';
+                const KEY_VALUE_SEPARATOR: char = '=';
+
                 let labels: Vec<Label> = labels
-                    .split(',')
-                    .flat_map(|label| label.split_once('='))
+                    .split(LABELS_SEPARATOR)
+                    .flat_map(|label| label.split_once(KEY_VALUE_SEPARATOR))
                     .map(|(name, value)| Label::new(
                         name.trim().to_string(), value.trim().to_string()
                     ))

From 9dec8612a24cdc4e3ace27899ae5b7a7bc85b613 Mon Sep 17 00:00:00 2001
From: Luca Cominardi <luca.cominardi@datadoghq.com>
Date: Mon, 3 Aug 2026 10:15:54 +0200
Subject: [PATCH 3/5] Rename metrics labels env var constant

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 quickwit/quickwit-metrics/src/inner.rs | 6 +++---
 quickwit/quickwit-metrics/src/lib.rs   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/quickwit/quickwit-metrics/src/inner.rs b/quickwit/quickwit-metrics/src/inner.rs
index ff02174ad44..407a56d7a6a 100644
--- a/quickwit/quickwit-metrics/src/inner.rs
+++ b/quickwit/quickwit-metrics/src/inner.rs
@@ -26,14 +26,14 @@ pub use const_format::concatcp as __concatcp;
 use metrics::Label;
 use rustc_hash::FxHasher;
 
-static METRICS_LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
+static LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
         // quickwit-common defines common helpers for getting environment variables.
         // However, we need to use the `std::env::var` function directly here because
         // quickwit-common depends on quickwit-metrics and it would cause a circular dependency.
 
         // The format of the environment variable is:
         // QW_METRICS_LABELS="environment=test,region=us-east-1,foo=bar"
-        match std::env::var(super::METRICS_LABELS_ENV_VAR_NAME) {
+        match std::env::var(super::QW_METRICS_LABELS_ENV_VAR) {
             Ok(labels) => {
                 const LABELS_SEPARATOR: char = ',';
                 const KEY_VALUE_SEPARATOR: char = '=';
@@ -54,7 +54,7 @@ static METRICS_LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
 /// Returns the labels from the environment variable.
 #[doc(hidden)]
 pub fn __labels_env_var() -> &'static [Label] {
-    METRICS_LABELS_ENV_VAR.as_ref()
+    LABELS_ENV_VAR.as_ref()
 }
 
 /// Counts the number of token-tree arguments at compile time.
diff --git a/quickwit/quickwit-metrics/src/lib.rs b/quickwit/quickwit-metrics/src/lib.rs
index 27eff481320..6108ff480f3 100644
--- a/quickwit/quickwit-metrics/src/lib.rs
+++ b/quickwit/quickwit-metrics/src/lib.rs
@@ -269,7 +269,7 @@ pub const SYSTEM: &str = "quickwit";
 pub const SEPARATOR: &str = "_";
 
 /// The name of the environment variable that contains the injected labels for all the metrics.
-pub(crate) const METRICS_LABELS_ENV_VAR_NAME: &str = "QW_METRICS_LABELS";
+pub(crate) const QW_METRICS_LABELS_ENV_VAR: &str = "QW_METRICS_LABELS";
 
 // ─── Metric modules ───
 mod counter;

From dd9b23215b8316b9ecda49257dddea94fb374722 Mon Sep 17 00:00:00 2001
From: Luca Cominardi <luca.cominardi@datadoghq.com>
Date: Mon, 3 Aug 2026 10:25:23 +0200
Subject: [PATCH 4/5] Move metrics labels docs to CLI reference

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/reference/cli.md     | 10 ++++++++++
 docs/reference/metrics.md | 10 ----------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 58477b6140f..5c9762d41da 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -876,3 +876,13 @@ RUST_LOG=debug quickwit run
 # run with log level info, except for indexing related logs
 RUST_LOG=info,quickwit_indexing=debug quickwit run
 ```
+
+### QW_METRICS_LABELS
+
+Attach the same labels to every Quickwit metric. Specify labels as a comma-separated list of `name=value` pairs.
+
+*Example*
+
+`QW_METRICS_LABELS="environment=production,region=us-east-1" quickwit run`
+
+The environment variable is read once, when the first metric is initialized. Set it before starting Quickwit; changing it while Quickwit is running has no effect.
diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md
index f740287c242..e75121f08f3 100644
--- a/docs/reference/metrics.md
+++ b/docs/reference/metrics.md
@@ -91,13 +91,3 @@ PostgreSQL-backed metastores also expose connection pool gauges:
 | `quickwit_storage` | `object_storage_puts_total` | Number of objects uploaded. May differ from object_storage_requests_parts due to multipart upload | `counter` |
 | `quickwit_storage` | `object_storage_puts_parts` | Number of object parts uploaded | `counter` |
 | `quickwit_storage` | `object_storage_download_num_bytes` | Amount of data downloaded from an object storage | `counter` |
-
-## Global labels
-
-You can attach the same labels to every Quickwit metric by setting the `QW_METRICS_LABELS` environment variable. Specify labels as a comma-separated list of `name=value` pairs:
-
-```bash
-QW_METRICS_LABELS="environment=production,region=us-east-1" quickwit run
-```
-
-The environment variable is read once, when the first metric is initialized. Set it before starting Quickwit; changing it while Quickwit is running has no effect.

From acd10c7524993af575d11a6db905c997be892c11 Mon Sep 17 00:00:00 2001
From: Luca Cominardi <luca.cominardi@datadoghq.com>
Date: Mon, 3 Aug 2026 12:07:27 +0200
Subject: [PATCH 5/5] Validate global metrics labels during telemetry startup

Reject malformed label configuration before installing recorders so metrics cannot start with silently incomplete global labels.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/reference/cli.md                         |   4 +-
 quickwit/Cargo.lock                           |   1 +
 quickwit/quickwit-metrics/Cargo.toml          |   1 +
 quickwit/quickwit-metrics/src/inner.rs        |  31 +----
 quickwit/quickwit-metrics/src/lib.rs          | 128 +++++++++++++++++-
 .../src/metrics.rs                            |   2 +
 6 files changed, 138 insertions(+), 29 deletions(-)

diff --git a/docs/reference/cli.md b/docs/reference/cli.md
index 5c9762d41da..e4cbaa5e1e3 100644
--- a/docs/reference/cli.md
+++ b/docs/reference/cli.md
@@ -879,10 +879,10 @@ RUST_LOG=info,quickwit_indexing=debug quickwit run
 
 ### QW_METRICS_LABELS
 
-Attach the same labels to every Quickwit metric. Specify labels as a comma-separated list of `name=value` pairs.
+Attach the same labels to every Quickwit metric. Specify labels as a comma-separated list of `name=value` pairs. Label names and values must not be empty.
 
 *Example*
 
 `QW_METRICS_LABELS="environment=production,region=us-east-1" quickwit run`
 
-The environment variable is read once, when the first metric is initialized. Set it before starting Quickwit; changing it while Quickwit is running has no effect.
+The environment variable is read once during telemetry initialization. Set it before starting Quickwit; changing it while Quickwit is running has no effect. If any entry does not use the `name=value` format, telemetry initialization fails.
diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock
index 78f4c0ae814..42d7f27bad8 100644
--- a/quickwit/Cargo.lock
+++ b/quickwit/Cargo.lock
@@ -9101,6 +9101,7 @@ dependencies = [
 name = "quickwit-metrics"
 version = "0.9.0"
 dependencies = [
+ "anyhow",
  "atomic_float",
  "const_format",
  "criterion",
diff --git a/quickwit/quickwit-metrics/Cargo.toml b/quickwit/quickwit-metrics/Cargo.toml
index 530149ae1d8..23b58740f2e 100644
--- a/quickwit/quickwit-metrics/Cargo.toml
+++ b/quickwit/quickwit-metrics/Cargo.toml
@@ -11,6 +11,7 @@ description = "Type-safe, zero-allocation metric declarations built on the metri
 categories = ["development-tools::profiling"]
 
 [dependencies]
+anyhow = { workspace = true }
 metrics = { workspace = true }
 metrics-util = { workspace = true }
 inventory = { workspace = true }
diff --git a/quickwit/quickwit-metrics/src/inner.rs b/quickwit/quickwit-metrics/src/inner.rs
index 407a56d7a6a..57d9ffb4039 100644
--- a/quickwit/quickwit-metrics/src/inner.rs
+++ b/quickwit/quickwit-metrics/src/inner.rs
@@ -19,42 +19,21 @@
 //! items via `$crate::`. **Do not use directly.**
 
 use std::hash::{Hash, Hasher};
-use std::sync::LazyLock;
 
 #[doc(hidden)]
 pub use const_format::concatcp as __concatcp;
 use metrics::Label;
 use rustc_hash::FxHasher;
 
-static LABELS_ENV_VAR: LazyLock<Box<[Label]>> = LazyLock::new(||
-        // quickwit-common defines common helpers for getting environment variables.
-        // However, we need to use the `std::env::var` function directly here because
-        // quickwit-common depends on quickwit-metrics and it would cause a circular dependency.
-
-        // The format of the environment variable is:
-        // QW_METRICS_LABELS="environment=test,region=us-east-1,foo=bar"
-        match std::env::var(super::QW_METRICS_LABELS_ENV_VAR) {
-            Ok(labels) => {
-                const LABELS_SEPARATOR: char = ',';
-                const KEY_VALUE_SEPARATOR: char = '=';
-
-                let labels: Vec<Label> = labels
-                    .split(LABELS_SEPARATOR)
-                    .flat_map(|label| label.split_once(KEY_VALUE_SEPARATOR))
-                    .map(|(name, value)| Label::new(
-                        name.trim().to_string(), value.trim().to_string()
-                    ))
-                    .collect();
-                labels.into_boxed_slice()
-            },
-            Err(_) => Vec::new().into_boxed_slice(),
-        });
+use crate::LABELS_ENV_VAR;
 
-// ─── Helper functions ───
 /// Returns the labels from the environment variable.
 #[doc(hidden)]
 pub fn __labels_env_var() -> &'static [Label] {
-    LABELS_ENV_VAR.as_ref()
+    match LABELS_ENV_VAR.get() {
+        Some(labels) => labels.as_ref(),
+        None => &[],
+    }
 }
 
 /// Counts the number of token-tree arguments at compile time.
diff --git a/quickwit/quickwit-metrics/src/lib.rs b/quickwit/quickwit-metrics/src/lib.rs
index 6108ff480f3..d2cb591d5f4 100644
--- a/quickwit/quickwit-metrics/src/lib.rs
+++ b/quickwit/quickwit-metrics/src/lib.rs
@@ -269,7 +269,8 @@ pub const SYSTEM: &str = "quickwit";
 pub const SEPARATOR: &str = "_";
 
 /// The name of the environment variable that contains the injected labels for all the metrics.
-pub(crate) const QW_METRICS_LABELS_ENV_VAR: &str = "QW_METRICS_LABELS";
+pub const QW_METRICS_LABELS_ENV_VAR: &str = "QW_METRICS_LABELS";
+pub(crate) static LABELS_ENV_VAR: OnceLock<Box<[Label]>> = OnceLock::new();
 
 // ─── Metric modules ───
 mod counter;
@@ -279,6 +280,8 @@ mod histogram;
 mod inner;
 mod labels;
 
+use std::sync::OnceLock;
+
 // ─── Internal helpers (re-exported for macro expansion) ───
 //
 // These re-exports exist so that downstream crates only need
@@ -308,6 +311,7 @@ pub use counter::{Counter, LazyCounter};
 pub use gauge::{Gauge, GaugeGuard, LazyGauge};
 pub use histogram::{Histogram, HistogramConfig, HistogramTimer, LazyHistogram};
 pub use labels::{LabelNames, Labels};
+use metrics::Label;
 // ─── metrics-rs re-exports ───
 pub use metrics::{CounterFn, GaugeFn, HistogramFn};
 pub use metrics_util::MetricKind;
@@ -385,3 +389,125 @@ pub fn histogram_buckets() -> impl Iterator<Item = (&'static str, Vec<f64>)> {
         (c.info.key_name, buckets)
     })
 }
+
+/// Initializes the global metrics labels from the environment variable.
+pub fn init_metrics_labels_env_var() -> anyhow::Result<()> {
+    // If the labels environment variable is already initialized, return early.
+    if LABELS_ENV_VAR.get().is_some() {
+        return Ok(());
+    }
+
+    // quickwit-common defines common helpers for getting environment variables.
+    // However, we need to use the `std::env::var` function directly here because
+    // quickwit-common depends on quickwit-metrics and it would cause a circular dependency.
+    let labels = match std::env::var(QW_METRICS_LABELS_ENV_VAR) {
+        Ok(labels) => labels,
+        Err(std::env::VarError::NotPresent) => String::new(),
+        Err(std::env::VarError::NotUnicode(_)) => {
+            anyhow::bail!("{QW_METRICS_LABELS_ENV_VAR} must contain valid Unicode")
+        }
+    };
+    let parsed_labels = parse_metrics_labels(&labels)?;
+    LABELS_ENV_VAR.get_or_init(|| parsed_labels.into_boxed_slice());
+
+    Ok(())
+}
+
+// The format of the environment variable is:
+// QW_METRICS_LABELS="environment=test,region=us-east-1,foo=bar"
+fn parse_metrics_labels(labels: &str) -> anyhow::Result<Vec<Label>> {
+    let mut parsed_labels: Vec<Label> = Vec::new();
+    if labels.trim().is_empty() {
+        return Ok(parsed_labels);
+    }
+
+    const LABELS_SEPARATOR: char = ',';
+    const KEY_VALUE_SEPARATOR: char = '=';
+
+    for label in labels.split(LABELS_SEPARATOR) {
+        let (name, value) = label.split_once(KEY_VALUE_SEPARATOR).ok_or_else(|| {
+            anyhow::anyhow!(
+                "{} contains invalid label format: {}",
+                QW_METRICS_LABELS_ENV_VAR,
+                label
+            )
+        })?;
+        let name = name.trim();
+        if name.is_empty() {
+            anyhow::bail!(
+                "{} contains an empty label name: {}",
+                QW_METRICS_LABELS_ENV_VAR,
+                label
+            );
+        }
+        let value = value.trim();
+        if value.is_empty() {
+            anyhow::bail!(
+                "{} contains an empty label value: {}",
+                QW_METRICS_LABELS_ENV_VAR,
+                label
+            );
+        }
+        let label = Label::new(name.to_string(), value.to_string());
+        parsed_labels.push(label);
+    }
+
+    Ok(parsed_labels)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_metrics_labels() {
+        let labels = parse_metrics_labels("environment=production, region=us-east-1 ")
+            .expect("labels should be valid");
+
+        assert_eq!(labels.len(), 2);
+        assert_eq!(labels[0].key(), "environment");
+        assert_eq!(labels[0].value(), "production");
+        assert_eq!(labels[1].key(), "region");
+        assert_eq!(labels[1].value(), "us-east-1");
+    }
+
+    #[test]
+    fn test_parse_metrics_labels_accepts_empty_input() {
+        let labels = parse_metrics_labels("  ").expect("empty input should be valid");
+
+        assert!(labels.is_empty());
+    }
+
+    #[test]
+    fn test_parse_metrics_labels_rejects_missing_separator() {
+        let error = parse_metrics_labels("environment=production,region")
+            .expect_err("label without a separator should be rejected");
+
+        assert_eq!(
+            error.to_string(),
+            "QW_METRICS_LABELS contains invalid label format: region"
+        );
+    }
+
+    #[test]
+    fn test_parse_metrics_labels_rejects_empty_name() {
+        let error =
+            parse_metrics_labels(" =production").expect_err("empty label name should be rejected");
+
+        assert_eq!(
+            error.to_string(),
+            "QW_METRICS_LABELS contains an empty label name:  =production"
+        );
+    }
+
+    #[test]
+    fn test_parse_metrics_labels_rejects_empty_value() {
+        let error = parse_metrics_labels("environment= ")
+            .expect_err("empty label value should be rejected");
+
+        assert_eq!(
+            error.to_string(),
+            "QW_METRICS_LABELS contains an empty label value: environment= "
+        );
+    }
+}
diff --git a/quickwit/quickwit-telemetry-exporters/src/metrics.rs b/quickwit/quickwit-telemetry-exporters/src/metrics.rs
index 68ec2fc2df4..0de9ced7f49 100644
--- a/quickwit/quickwit-telemetry-exporters/src/metrics.rs
+++ b/quickwit/quickwit-telemetry-exporters/src/metrics.rs
@@ -22,6 +22,8 @@ pub(crate) fn init_metrics_provider(
     service_version: &str,
     otlp_config: &OtlpExporterConfig,
 ) -> anyhow::Result<Option<SdkMeterProvider>> {
+    quickwit_metrics::init_metrics_labels_env_var()?;
+
     let prometheus_recorder = crate::prometheus::metrics::build_recorder()?;
 
     let (recorder, meter_provider) = if otlp_config.is_enabled() {