feat(telemetry): export per-deployment metrics and logs over OTLP

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - OpenTelemetry distributed tracing (opt-in) via `[server.telemetry.traces]`: `ring server start` exports spans over OTLP/gRPC — one per HTTP request (stable HTTP-server semantic-convention attributes) and one per productive scheduler cycle. Endpoint, `service.name` and sampler are configurable, standard `OTEL_*` env vars override the TOML, and an unreachable collector degrades gracefully instead of failing the server
+- OpenTelemetry metric export (opt-in, push) via `[server.telemetry.metrics]`: periodically pushes the per-deployment resource gauges (CPU, memory, network, disk, PIDs, instance and restart counts, labelled `{deployment, namespace, runtime}`) over OTLP/gRPC, read from the same stats cache the Prometheus `/metrics` endpoint uses so no extra runtime round-trip is added. Endpoint, `service.name` and push interval are configurable
+- OpenTelemetry log export (opt-in, push) via `[server.telemetry.logs]`: bridges the server's log events to an OTLP/gRPC collector in addition to the console. Each telemetry signal (traces, metrics, logs) is independently toggled; all three degrade gracefully when the collector is unreachable
 
 ## [0.10.0] - 2026-07-04
 

Cargo.toml

@@ -85,17 +85,21 @@ prost = "0.14"
 prost-types = "0.14"
 tokio-stream = "0.1"
 docker_credential = "1.4.0"
-# OpenTelemetry distributed tracing. Spans from `tracing` are bridged to OTel
-# via `tracing-opentelemetry` and exported over OTLP/gRPC (tonic transport,
-# already vendored above for containerd — one tonic across the tree). The whole
-# pipeline is opt-in: with `[server.telemetry.traces]` disabled no exporter is
-# built and there is zero runtime cost. Pinned in lockstep on 0.32 (the core
-# crates release together); `tracing-opentelemetry` 0.33 targets opentelemetry
-# 0.32.
+# OpenTelemetry export. Three signals, all opt-in per sub-block under
+# `[server.telemetry]`, all over OTLP/gRPC (tonic transport, already vendored
+# above for containerd — one tonic across the tree):
+#   - traces: spans from `tracing` bridged via `tracing-opentelemetry`;
+#   - metrics: the per-deployment runtime gauges (read from the stats cache,
+#     no extra runtime round-trip) pushed via an `SdkMeterProvider`;
+#   - logs: `tracing` events bridged to OTLP via `opentelemetry-appender-tracing`.
+# With a signal disabled (the default) no exporter is built and there is zero
+# runtime cost. Pinned in lockstep on 0.32 (the core crates release together);
+# `tracing-opentelemetry` 0.33 targets opentelemetry 0.32.
 opentelemetry = "0.32"
 opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace", "metrics", "logs"] }
 tracing-opentelemetry = "0.33"
+opentelemetry-appender-tracing = "0.32"
 
 [dev-dependencies]
 axum-test = "17.3.0"

documentation/reference/config-toml.md

@@ -16,6 +16,8 @@ A context describes one client→server connection; it has no business deciding
 [server.scheduler]                        # optional
 [server.dashboard]                        # optional
 [server.telemetry.traces]                 # opt-in: enabled = true
+[server.telemetry.metrics]                # opt-in: enabled = true
+[server.telemetry.logs]                   # opt-in: enabled = true
 [server.runtime.docker]                   # opt-in: enabled = true
 [server.runtime.cloud_hypervisor]         # opt-in: enabled = true
 
@@ -91,6 +93,33 @@ Standard `OTEL_*` environment variables take precedence over the TOML values, so
 
 Ring emits one span per HTTP request (named `{method} {route}` with the stable HTTP-server semantic-convention attributes) and one `scheduler.cycle` span per reconciliation cycle that has work to do — idle scheduler ticks produce no spans.
 
+Each telemetry signal is independent: you can enable traces, metrics, or logs on their own or in any combination. All three are off by default.
+
+### `[server.telemetry.metrics]`
+
+Opt-in OpenTelemetry metric export (push) over OTLP/gRPC. Off by default. When enabled, Ring periodically pushes the same per-deployment resource series the Prometheus `/metrics` endpoint serves, read from the background stats cache (so enabling it adds no extra runtime round-trip). Only `ring server start` exports metrics.
+
+| Field | Type | Required | Default | Purpose |
+|---|---|---|---|---|
+| `enabled` | bool | no | `false` | Push metrics to an OTLP/gRPC collector |
+| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`) |
+| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` |
+| `interval_seconds` | int | no | `15` | Push interval. Floored at 1s |
+
+The exported gauges, labelled `{deployment, namespace, runtime}`, are: `ring.deployment.cpu.percent`, `ring.deployment.memory.used_bytes`, `ring.deployment.memory.limit_bytes`, `ring.deployment.network.rx_bytes`, `ring.deployment.network.tx_bytes`, `ring.deployment.disk.read_bytes`, `ring.deployment.disk.write_bytes`, `ring.deployment.pids`, `ring.deployment.instances`, and `ring.deployment.restarts`. Values are at most one stats-cache refresh (the scheduler interval) stale. A failed exporter is logged and the server continues without OTLP metrics.
+
+### `[server.telemetry.logs]`
+
+Opt-in OpenTelemetry log export (push) over OTLP/gRPC. Off by default. When enabled, Ring's log events are bridged to an OTLP collector **in addition to** the console — console logging is never turned off. Only `ring server start` exports logs.
+
+| Field | Type | Required | Default | Purpose |
+|---|---|---|---|---|
+| `enabled` | bool | no | `false` | Export log records to an OTLP/gRPC collector |
+| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`) |
+| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` |
+
+A failed exporter is logged and the server continues console-only rather than failing.
+
 ### `[server.runtime.docker]`
 
 | Field | Type | Required | Default | Purpose |

src/commands/server.rs

@@ -29,7 +29,11 @@ pub(crate) fn command_config() -> Command {
     )
 }
 
-pub(crate) async fn execute(args: &ArgMatches, mut configuration: Config) {
+pub(crate) async fn execute(
+    args: &ArgMatches,
+    mut configuration: Config,
+    telemetry_guard: Option<&mut crate::telemetry::OtelGuard>,
+) {
     // Dashboard activation precedence, strongest first:
     //   1. `--dashboard` CLI flag (explicit one-off)
     //   2. `RING_DASHBOARD=true|1|yes` env var (systemd / docker / CI)
@@ -308,6 +312,30 @@ pub(crate) async fn execute(args: &ArgMatches, mut configuration: Config) {
     // background and read by `/metrics` so a scrape never blocks on the
     // runtimes. See `scheduler::stats_cache`.
     let stats_cache = crate::scheduler::stats_cache::new_cache();
+
+    // OTLP metrics push (opt-in). The meter provider is built here — not in
+    // `init_tracing` — because its observable gauges read this stats cache, and
+    // it reports the same per-deployment series the Prometheus `/metrics`
+    // endpoint serves, so enabling it adds no runtime round-trip. Non-fatal: a
+    // failed exporter is logged and the server continues without OTLP metrics.
+    if let Some(guard) = telemetry_guard {
+        let metrics_cfg = &configuration.server.telemetry.metrics;
+        if metrics_cfg.enabled {
+            match crate::telemetry::build_meter(metrics_cfg, stats_cache.clone()) {
+                Ok(provider) => {
+                    guard.attach_meter(provider);
+                    info!(
+                        "telemetry: OTLP metrics export enabled ({}, every {}s)",
+                        metrics_cfg.endpoint,
+                        metrics_cfg.resolved_interval_seconds()
+                    );
+                }
+                Err(e) => error!(
+                    "telemetry: failed to initialise metrics exporter, continuing without OTLP metrics: {e}"
+                ),
+            }
+        }
+    }
     {
         let cache = stats_cache.clone();
         let cache_pool = pool.clone();

src/config/server.rs

@@ -276,14 +276,19 @@ impl Default for DashboardConfig {
 }
 
 /// `[server.telemetry]` — OpenTelemetry export. One sub-block per signal so the
-/// table can grow without breaking existing configs: `traces` ships now; `logs`
-/// and `metrics` are reserved for later phases and are intentionally absent from
-/// the struct until implemented (an unknown `[server.telemetry.logs]` table is
-/// simply ignored by serde today, so adding it later is backward-compatible).
+/// table can grow without breaking existing configs. Each signal is independent
+/// and opt-in: `traces` (spans), `metrics` (per-deployment runtime gauges), and
+/// `logs` (server log events) can each be enabled on their own, and with all
+/// three disabled (the default) no exporter is built and Ring runs exactly as
+/// before.
 #[derive(Deserialize, Debug, Clone, Default)]
 pub(crate) struct TelemetryConfig {
     #[serde(default)]
     pub(crate) traces: TracesConfig,
+    #[serde(default)]
+    pub(crate) metrics: MetricsConfig,
+    #[serde(default)]
+    pub(crate) logs: LogsConfig,
 }
 
 /// `[server.telemetry.traces]` — OTLP/gRPC span export. Opt-in: with `enabled`
@@ -350,6 +355,131 @@ impl TracesConfig {
     }
 }
 
+/// `[server.telemetry.metrics]` — OTLP/gRPC metric export (push). Opt-in: with
+/// `enabled` false (the default) no meter provider is built. When on, an
+/// `SdkMeterProvider` periodically exports the same per-deployment runtime
+/// gauges the Prometheus `/metrics` endpoint already serves (CPU, memory,
+/// network, disk, PIDs, instance count), read from the background stats cache
+/// so no extra runtime round-trip is added.
+///
+/// `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT`
+/// override `endpoint`; `OTEL_SERVICE_NAME` overrides `service_name`.
+#[derive(Deserialize, Debug, Clone)]
+pub(crate) struct MetricsConfig {
+    #[serde(default)]
+    pub(crate) enabled: bool,
+    /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`.
+    #[serde(default = "default_metrics_endpoint")]
+    pub(crate) endpoint: String,
+    /// `service.name` resource attribute reported to the collector.
+    #[serde(default = "default_metrics_service_name")]
+    pub(crate) service_name: String,
+    /// Push interval in seconds. The meter provider reads the stats snapshot and
+    /// exports on this cadence; the values are at most one stats-cache refresh
+    /// stale regardless (the cache refreshes on the scheduler interval).
+    #[serde(default = "default_metrics_interval_seconds")]
+    pub(crate) interval_seconds: u64,
+}
+
+fn default_metrics_endpoint() -> String {
+    "http://127.0.0.1:4317".to_string()
+}
+
+fn default_metrics_service_name() -> String {
+    "ring-server".to_string()
+}
+
+fn default_metrics_interval_seconds() -> u64 {
+    15
+}
+
+impl Default for MetricsConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            endpoint: default_metrics_endpoint(),
+            service_name: default_metrics_service_name(),
+            interval_seconds: default_metrics_interval_seconds(),
+        }
+    }
+}
+
+impl MetricsConfig {
+    /// Effective collector endpoint: the metrics-specific
+    /// `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` wins, then the shared
+    /// `OTEL_EXPORTER_OTLP_ENDPOINT`, then the configured value.
+    pub(crate) fn resolved_endpoint(&self) -> String {
+        std::env::var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
+            .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT"))
+            .unwrap_or_else(|_| self.endpoint.clone())
+    }
+
+    /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured
+    /// value.
+    pub(crate) fn resolved_service_name(&self) -> String {
+        std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone())
+    }
+
+    /// Push interval, floored at 1s so a misconfigured `0` cannot spin.
+    pub(crate) fn resolved_interval_seconds(&self) -> u64 {
+        self.interval_seconds.max(1)
+    }
+}
+
+/// `[server.telemetry.logs]` — OTLP/gRPC log export (push). Opt-in: with
+/// `enabled` false (the default) no logger provider is built and logs go only
+/// to the console. When on, `tracing` events are bridged to an
+/// `SdkLoggerProvider` and exported over OTLP, in addition to the console.
+///
+/// `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT`
+/// override `endpoint`; `OTEL_SERVICE_NAME` overrides `service_name`.
+#[derive(Deserialize, Debug, Clone)]
+pub(crate) struct LogsConfig {
+    #[serde(default)]
+    pub(crate) enabled: bool,
+    /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`.
+    #[serde(default = "default_logs_endpoint")]
+    pub(crate) endpoint: String,
+    /// `service.name` resource attribute reported to the collector.
+    #[serde(default = "default_logs_service_name")]
+    pub(crate) service_name: String,
+}
+
+fn default_logs_endpoint() -> String {
+    "http://127.0.0.1:4317".to_string()
+}
+
+fn default_logs_service_name() -> String {
+    "ring-server".to_string()
+}
+
+impl Default for LogsConfig {
+    fn default() -> Self {
+        Self {
+            enabled: false,
+            endpoint: default_logs_endpoint(),
+            service_name: default_logs_service_name(),
+        }
+    }
+}
+
+impl LogsConfig {
+    /// Effective collector endpoint: the logs-specific
+    /// `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` wins, then the shared
+    /// `OTEL_EXPORTER_OTLP_ENDPOINT`, then the configured value.
+    pub(crate) fn resolved_endpoint(&self) -> String {
+        std::env::var("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
+            .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT"))
+            .unwrap_or_else(|_| self.endpoint.clone())
+    }
+
+    /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured
+    /// value.
+    pub(crate) fn resolved_service_name(&self) -> String {
+        std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone())
+    }
+}
+
 #[cfg(test)]
 mod telemetry_tests {
     use super::*;
@@ -364,10 +494,75 @@ mod telemetry_tests {
     }
 
     #[test]
-    fn telemetry_absent_from_toml_yields_disabled_traces() {
+    fn telemetry_absent_from_toml_yields_disabled_signals() {
         // A `[server]` table with no telemetry block must not enable anything.
         let cfg: ServerConfig = toml::from_str("").unwrap();
         assert!(!cfg.telemetry.traces.enabled);
+        assert!(!cfg.telemetry.metrics.enabled);
+        assert!(!cfg.telemetry.logs.enabled);
+    }
+
+    #[test]
+    fn metrics_disabled_by_default() {
+        let c = MetricsConfig::default();
+        assert!(!c.enabled);
+        assert_eq!(c.endpoint, "http://127.0.0.1:4317");
+        assert_eq!(c.service_name, "ring-server");
+        assert_eq!(c.interval_seconds, 15);
+    }
+
+    #[test]
+    fn logs_disabled_by_default() {
+        let c = LogsConfig::default();
+        assert!(!c.enabled);
+        assert_eq!(c.endpoint, "http://127.0.0.1:4317");
+        assert_eq!(c.service_name, "ring-server");
+    }
+
+    #[test]
+    fn metrics_and_logs_blocks_parse_from_toml() {
+        let cfg: ServerConfig = toml::from_str(
+            r#"
+            [telemetry.metrics]
+            enabled = true
+            endpoint = "http://collector:4317"
+            interval_seconds = 30
+
+            [telemetry.logs]
+            enabled = true
+            endpoint = "http://collector:4318"
+            "#,
+        )
+        .unwrap();
+        assert!(cfg.telemetry.metrics.enabled);
+        assert_eq!(cfg.telemetry.metrics.endpoint, "http://collector:4317");
+        assert_eq!(cfg.telemetry.metrics.interval_seconds, 30);
+        assert!(cfg.telemetry.logs.enabled);
+        assert_eq!(cfg.telemetry.logs.endpoint, "http://collector:4318");
+    }
+
+    #[test]
+    fn metrics_interval_zero_is_floored_to_one() {
+        let c = MetricsConfig {
+            interval_seconds: 0,
+            ..Default::default()
+        };
+        assert_eq!(c.resolved_interval_seconds(), 1);
+    }
+
+    #[test]
+    fn signals_are_independent() {
+        // Enabling metrics alone must leave traces and logs off.
+        let cfg: ServerConfig = toml::from_str(
+            r#"
+            [telemetry.metrics]
+            enabled = true
+            "#,
+        )
+        .unwrap();
+        assert!(cfg.telemetry.metrics.enabled);
+        assert!(!cfg.telemetry.traces.enabled);
+        assert!(!cfg.telemetry.logs.enabled);
     }
 
     #[test]