From d8d7a58173ea32a3a31203c1090bfc3b23252eae Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Sat, 4 Jul 2026 15:12:40 +0300 Subject: [PATCH 1/3] feat(telemetry): add opt-in metrics and logs config blocks --- Cargo.lock | 13 +++ Cargo.toml | 20 +++-- src/config/server.rs | 205 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 225 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbcfbf1..53733ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2035,6 +2035,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "opentelemetry-otlp" version = "0.32.0" @@ -2584,6 +2596,7 @@ dependencies = [ "nix", "once_cell", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", "owo-colors", diff --git a/Cargo.toml b/Cargo.toml index 0ac33f3..d554b14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,17 +85,21 @@ prost = "0.14" prost-types = "0.14" tokio-stream = "0.1" docker_credential = "1.4.0" -# OpenTelemetry distributed tracing. Spans from `tracing` are bridged to OTel -# via `tracing-opentelemetry` and exported over OTLP/gRPC (tonic transport, -# already vendored above for containerd — one tonic across the tree). The whole -# pipeline is opt-in: with `[server.telemetry.traces]` disabled no exporter is -# built and there is zero runtime cost. Pinned in lockstep on 0.32 (the core -# crates release together); `tracing-opentelemetry` 0.33 targets opentelemetry -# 0.32. +# OpenTelemetry export. Three signals, all opt-in per sub-block under +# `[server.telemetry]`, all over OTLP/gRPC (tonic transport, already vendored +# above for containerd — one tonic across the tree): +# - traces: spans from `tracing` bridged via `tracing-opentelemetry`; +# - metrics: the per-deployment runtime gauges (read from the stats cache, +# no extra runtime round-trip) pushed via an `SdkMeterProvider`; +# - logs: `tracing` events bridged to OTLP via `opentelemetry-appender-tracing`. +# With a signal disabled (the default) no exporter is built and there is zero +# runtime cost. Pinned in lockstep on 0.32 (the core crates release together); +# `tracing-opentelemetry` 0.33 targets opentelemetry 0.32. opentelemetry = "0.32" opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace", "metrics", "logs"] } tracing-opentelemetry = "0.33" +opentelemetry-appender-tracing = "0.32" [dev-dependencies] axum-test = "17.3.0" diff --git a/src/config/server.rs b/src/config/server.rs index 16bd31d..819e517 100644 --- a/src/config/server.rs +++ b/src/config/server.rs @@ -276,14 +276,19 @@ impl Default for DashboardConfig { } /// `[server.telemetry]` — OpenTelemetry export. One sub-block per signal so the -/// table can grow without breaking existing configs: `traces` ships now; `logs` -/// and `metrics` are reserved for later phases and are intentionally absent from -/// the struct until implemented (an unknown `[server.telemetry.logs]` table is -/// simply ignored by serde today, so adding it later is backward-compatible). +/// table can grow without breaking existing configs. Each signal is independent +/// and opt-in: `traces` (spans), `metrics` (per-deployment runtime gauges), and +/// `logs` (server log events) can each be enabled on their own, and with all +/// three disabled (the default) no exporter is built and Ring runs exactly as +/// before. #[derive(Deserialize, Debug, Clone, Default)] pub(crate) struct TelemetryConfig { #[serde(default)] pub(crate) traces: TracesConfig, + #[serde(default)] + pub(crate) metrics: MetricsConfig, + #[serde(default)] + pub(crate) logs: LogsConfig, } /// `[server.telemetry.traces]` — OTLP/gRPC span export. Opt-in: with `enabled` @@ -350,6 +355,131 @@ impl TracesConfig { } } +/// `[server.telemetry.metrics]` — OTLP/gRPC metric export (push). Opt-in: with +/// `enabled` false (the default) no meter provider is built. When on, an +/// `SdkMeterProvider` periodically exports the same per-deployment runtime +/// gauges the Prometheus `/metrics` endpoint already serves (CPU, memory, +/// network, disk, PIDs, instance count), read from the background stats cache +/// so no extra runtime round-trip is added. +/// +/// `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` +/// override `endpoint`; `OTEL_SERVICE_NAME` overrides `service_name`. +#[derive(Deserialize, Debug, Clone)] +pub(crate) struct MetricsConfig { + #[serde(default)] + pub(crate) enabled: bool, + /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`. + #[serde(default = "default_metrics_endpoint")] + pub(crate) endpoint: String, + /// `service.name` resource attribute reported to the collector. + #[serde(default = "default_metrics_service_name")] + pub(crate) service_name: String, + /// Push interval in seconds. The meter provider reads the stats snapshot and + /// exports on this cadence; the values are at most one stats-cache refresh + /// stale regardless (the cache refreshes on the scheduler interval). + #[serde(default = "default_metrics_interval_seconds")] + pub(crate) interval_seconds: u64, +} + +fn default_metrics_endpoint() -> String { + "http://127.0.0.1:4317".to_string() +} + +fn default_metrics_service_name() -> String { + "ring-server".to_string() +} + +fn default_metrics_interval_seconds() -> u64 { + 15 +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + enabled: false, + endpoint: default_metrics_endpoint(), + service_name: default_metrics_service_name(), + interval_seconds: default_metrics_interval_seconds(), + } + } +} + +impl MetricsConfig { + /// Effective collector endpoint: the metrics-specific + /// `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` wins, then the shared + /// `OTEL_EXPORTER_OTLP_ENDPOINT`, then the configured value. + pub(crate) fn resolved_endpoint(&self) -> String { + std::env::var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") + .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")) + .unwrap_or_else(|_| self.endpoint.clone()) + } + + /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured + /// value. + pub(crate) fn resolved_service_name(&self) -> String { + std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone()) + } + + /// Push interval, floored at 1s so a misconfigured `0` cannot spin. + pub(crate) fn resolved_interval_seconds(&self) -> u64 { + self.interval_seconds.max(1) + } +} + +/// `[server.telemetry.logs]` — OTLP/gRPC log export (push). Opt-in: with +/// `enabled` false (the default) no logger provider is built and logs go only +/// to the console. When on, `tracing` events are bridged to an +/// `SdkLoggerProvider` and exported over OTLP, in addition to the console. +/// +/// `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` then `OTEL_EXPORTER_OTLP_ENDPOINT` +/// override `endpoint`; `OTEL_SERVICE_NAME` overrides `service_name`. +#[derive(Deserialize, Debug, Clone)] +pub(crate) struct LogsConfig { + #[serde(default)] + pub(crate) enabled: bool, + /// OTLP/gRPC collector endpoint, e.g. `http://collector:4317`. + #[serde(default = "default_logs_endpoint")] + pub(crate) endpoint: String, + /// `service.name` resource attribute reported to the collector. + #[serde(default = "default_logs_service_name")] + pub(crate) service_name: String, +} + +fn default_logs_endpoint() -> String { + "http://127.0.0.1:4317".to_string() +} + +fn default_logs_service_name() -> String { + "ring-server".to_string() +} + +impl Default for LogsConfig { + fn default() -> Self { + Self { + enabled: false, + endpoint: default_logs_endpoint(), + service_name: default_logs_service_name(), + } + } +} + +impl LogsConfig { + /// Effective collector endpoint: the logs-specific + /// `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` wins, then the shared + /// `OTEL_EXPORTER_OTLP_ENDPOINT`, then the configured value. + pub(crate) fn resolved_endpoint(&self) -> String { + std::env::var("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") + .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")) + .unwrap_or_else(|_| self.endpoint.clone()) + } + + /// Effective `service.name`: `OTEL_SERVICE_NAME` overrides the configured + /// value. + pub(crate) fn resolved_service_name(&self) -> String { + std::env::var("OTEL_SERVICE_NAME").unwrap_or_else(|_| self.service_name.clone()) + } +} + #[cfg(test)] mod telemetry_tests { use super::*; @@ -364,10 +494,75 @@ mod telemetry_tests { } #[test] - fn telemetry_absent_from_toml_yields_disabled_traces() { + fn telemetry_absent_from_toml_yields_disabled_signals() { // A `[server]` table with no telemetry block must not enable anything. let cfg: ServerConfig = toml::from_str("").unwrap(); assert!(!cfg.telemetry.traces.enabled); + assert!(!cfg.telemetry.metrics.enabled); + assert!(!cfg.telemetry.logs.enabled); + } + + #[test] + fn metrics_disabled_by_default() { + let c = MetricsConfig::default(); + assert!(!c.enabled); + assert_eq!(c.endpoint, "http://127.0.0.1:4317"); + assert_eq!(c.service_name, "ring-server"); + assert_eq!(c.interval_seconds, 15); + } + + #[test] + fn logs_disabled_by_default() { + let c = LogsConfig::default(); + assert!(!c.enabled); + assert_eq!(c.endpoint, "http://127.0.0.1:4317"); + assert_eq!(c.service_name, "ring-server"); + } + + #[test] + fn metrics_and_logs_blocks_parse_from_toml() { + let cfg: ServerConfig = toml::from_str( + r#" + [telemetry.metrics] + enabled = true + endpoint = "http://collector:4317" + interval_seconds = 30 + + [telemetry.logs] + enabled = true + endpoint = "http://collector:4318" + "#, + ) + .unwrap(); + assert!(cfg.telemetry.metrics.enabled); + assert_eq!(cfg.telemetry.metrics.endpoint, "http://collector:4317"); + assert_eq!(cfg.telemetry.metrics.interval_seconds, 30); + assert!(cfg.telemetry.logs.enabled); + assert_eq!(cfg.telemetry.logs.endpoint, "http://collector:4318"); + } + + #[test] + fn metrics_interval_zero_is_floored_to_one() { + let c = MetricsConfig { + interval_seconds: 0, + ..Default::default() + }; + assert_eq!(c.resolved_interval_seconds(), 1); + } + + #[test] + fn signals_are_independent() { + // Enabling metrics alone must leave traces and logs off. + let cfg: ServerConfig = toml::from_str( + r#" + [telemetry.metrics] + enabled = true + "#, + ) + .unwrap(); + assert!(cfg.telemetry.metrics.enabled); + assert!(!cfg.telemetry.traces.enabled); + assert!(!cfg.telemetry.logs.enabled); } #[test] From acb9fbf6e8d6bbe94549d9c49658dd9136cf240e Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Sat, 4 Jul 2026 15:12:47 +0300 Subject: [PATCH 2/3] feat(telemetry): export per-deployment metrics and logs over OTLP --- src/commands/server.rs | 30 +++- src/main.rs | 112 +++++++++++---- src/telemetry.rs | 301 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 390 insertions(+), 53 deletions(-) diff --git a/src/commands/server.rs b/src/commands/server.rs index ebaef50..ef59212 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -29,7 +29,11 @@ pub(crate) fn command_config() -> Command { ) } -pub(crate) async fn execute(args: &ArgMatches, mut configuration: Config) { +pub(crate) async fn execute( + args: &ArgMatches, + mut configuration: Config, + telemetry_guard: Option<&mut crate::telemetry::OtelGuard>, +) { // Dashboard activation precedence, strongest first: // 1. `--dashboard` CLI flag (explicit one-off) // 2. `RING_DASHBOARD=true|1|yes` env var (systemd / docker / CI) @@ -308,6 +312,30 @@ pub(crate) async fn execute(args: &ArgMatches, mut configuration: Config) { // background and read by `/metrics` so a scrape never blocks on the // runtimes. See `scheduler::stats_cache`. let stats_cache = crate::scheduler::stats_cache::new_cache(); + + // OTLP metrics push (opt-in). The meter provider is built here — not in + // `init_tracing` — because its observable gauges read this stats cache, and + // it reports the same per-deployment series the Prometheus `/metrics` + // endpoint serves, so enabling it adds no runtime round-trip. Non-fatal: a + // failed exporter is logged and the server continues without OTLP metrics. + if let Some(guard) = telemetry_guard { + let metrics_cfg = &configuration.server.telemetry.metrics; + if metrics_cfg.enabled { + match crate::telemetry::build_meter(metrics_cfg, stats_cache.clone()) { + Ok(provider) => { + guard.attach_meter(provider); + info!( + "telemetry: OTLP metrics export enabled ({}, every {}s)", + metrics_cfg.endpoint, + metrics_cfg.resolved_interval_seconds() + ); + } + Err(e) => error!( + "telemetry: failed to initialise metrics exporter, continuing without OTLP metrics: {e}" + ), + } + } + } { let cache = stats_cache.clone(); let cache_pool = pool.clone(); diff --git a/src/main.rs b/src/main.rs index 57f20fb..32c22b8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,39 +41,97 @@ fn env_filter() -> tracing_subscriber::EnvFilter { } /// Initialise `tracing`. Always installs the console `fmt` layer; when -/// `traces` is `Some` and enabled, also attaches the OpenTelemetry span layer -/// and returns its guard (which must be kept alive for the process). If the -/// OTLP exporter can't be built, logs the error and continues console-only. +/// `telemetry` is `Some` (server start) it also attaches whichever of the +/// subscriber-layer signals — traces and logs — are enabled, returning a guard +/// that owns their providers (kept alive for the process). Metrics is not a +/// subscriber layer; it is built later, once the stats cache exists, and +/// attached to the same guard via [`telemetry::OtelGuard::attach_meter`]. /// -/// Called exactly once, after the config is loaded, so `server start` can turn -/// tracing on while every other command stays console-only at zero cost. -fn init_tracing(traces: Option<&config::server::TracesConfig>) -> Option { +/// Each signal is independent and non-fatal: if an OTLP exporter can't be built +/// the error is logged and that signal is skipped, but the console subscriber +/// (and the other signals) still come up. Called exactly once, after the config +/// is loaded, so every non-server command stays console-only at zero cost. +fn init_tracing( + telemetry: Option<&config::server::TelemetryConfig>, +) -> Option { use tracing_subscriber::layer::{Layer, SubscriberExt}; use tracing_subscriber::util::SubscriberInitExt; let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter()); let registry = tracing_subscriber::registry().with(fmt_layer); - match traces { - Some(cfg) if cfg.enabled => match telemetry::build_tracer(cfg) { - Ok((otel_layer, guard)) => { - registry.with(otel_layer.with_filter(env_filter())).init(); - info!("telemetry: OTLP trace export enabled ({})", cfg.endpoint); - Some(guard) + let Some(cfg) = telemetry else { + registry.init(); + return None; + }; + + let mut guard = telemetry::OtelGuard::default(); + + // Outcome messages are buffered and emitted *after* `.init()`: logging here + // would go nowhere because no subscriber is active until the registry is + // initialised. `(is_error, message)` — errors are rare (a bad exporter) but + // must still surface once logging is live. + let mut messages: Vec<(bool, String)> = Vec::new(); + + // Traces: an OTel span layer over the registry. + let trace_layer = if cfg.traces.enabled { + match telemetry::build_tracer(&cfg.traces) { + Ok((layer, tracer_guard)) => { + guard.absorb(tracer_guard); + messages.push(( + false, + format!( + "telemetry: OTLP trace export enabled ({})", + cfg.traces.endpoint + ), + )); + Some(layer.with_filter(env_filter())) } Err(e) => { - registry.init(); - error!( - "telemetry: failed to initialise OTLP exporter, continuing without traces: {e}" - ); + messages.push((true, format!( + "telemetry: failed to initialise trace exporter, continuing without traces: {e}" + ))); + None + } + } + } else { + None + }; + + // Logs: a tracing→OTLP bridge layer, in addition to the console. + let log_layer = if cfg.logs.enabled { + match telemetry::build_logger(&cfg.logs) { + Ok((layer, provider)) => { + guard.attach_logger(provider); + messages.push(( + false, + format!("telemetry: OTLP log export enabled ({})", cfg.logs.endpoint), + )); + Some(layer.with_filter(env_filter())) + } + Err(e) => { + messages.push((true, format!( + "telemetry: failed to initialise log exporter, continuing without OTLP logs: {e}" + ))); None } - }, - _ => { - registry.init(); - None + } + } else { + None + }; + + registry.with(trace_layer).with(log_layer).init(); + + // Now that the subscriber is live, replay the buffered outcomes. + for (is_error, msg) in messages { + if is_error { + error!("{msg}"); + } else { + info!("{msg}"); } } + + Some(guard) } #[tokio::main] @@ -192,15 +250,17 @@ async fn main() { let subcommand_name = matches.subcommand(); let config = config::config::load_config(context, config_file); - // OTLP trace export is a server concern: enable it only for `server start`, - // where the daemon runs long enough for batching to matter. Every other - // command initialises console logging only. The guard lives for the whole - // of `main` so spans flush on exit. + // OTLP export is a server concern: enable it only for `server start`, where + // the daemon runs long enough for batching to matter. Every other command + // initialises console logging only. The guard lives for the whole of `main` + // so every signal flushes on exit. Traces and logs are installed now (they + // are subscriber layers); metrics is attached later, once the stats cache + // exists (see `server::execute`). let is_server_start = matches!( subcommand_name, Some(("server", sub)) if sub.subcommand().map(|(n, _)| n).unwrap_or("start") == "start" ); - let _telemetry_guard = init_tracing(is_server_start.then_some(&config.server.telemetry.traces)); + let mut telemetry_guard = init_tracing(is_server_start.then_some(&config.server.telemetry)); let client = reqwest::Client::builder() .default_headers({ @@ -221,7 +281,7 @@ async fn main() { Some(("server", sub_matches)) => { let server_command = sub_matches.subcommand().unwrap_or(("start", sub_matches)); if let ("start", sub_matches) = server_command { - commands::server::execute(sub_matches, config).await + commands::server::execute(sub_matches, config, telemetry_guard.as_mut()).await } } Some(("apply", sub_matches)) => { diff --git a/src/telemetry.rs b/src/telemetry.rs index a0b70f6..45501f7 100644 --- a/src/telemetry.rs +++ b/src/telemetry.rs @@ -1,30 +1,44 @@ -//! OpenTelemetry distributed-tracing pipeline (opt-in). +//! OpenTelemetry export pipeline (opt-in, one signal per sub-block). //! -//! When `[server.telemetry.traces]` is enabled, [`build_tracer`] builds an -//! OTLP/gRPC span exporter, wires it into an [`SdkTracerProvider`] with a -//! configurable sampler, and returns a `tracing_subscriber` layer that turns -//! every `tracing` span into an OpenTelemetry span, plus a guard. +//! Three independent signals, each built only when its `[server.telemetry.*]` +//! block is enabled: //! -//! The guard ([`OtelGuard`]) owns the provider and flushes on drop: dropping it -//! without flushing would lose the last, un-exported batch. Keep it alive for -//! the whole process (bind it in `main`), and it shuts the pipeline down cleanly -//! on exit. +//! - **traces** — [`build_tracer`] builds an OTLP/gRPC span exporter behind an +//! [`SdkTracerProvider`] with a configurable sampler, and returns a +//! `tracing_subscriber` layer that turns every `tracing` span into an OTel +//! span. +//! - **metrics** — [`build_meter`] builds an [`SdkMeterProvider`] with a +//! periodic OTLP push reader and an observable callback that reads the +//! background stats cache (no extra runtime round-trip) and reports the +//! per-deployment resource gauges the Prometheus `/metrics` endpoint already +//! serves. +//! - **logs** — [`build_logger`] builds an [`SdkLoggerProvider`] and returns an +//! `opentelemetry-appender-tracing` layer that ships `tracing` events over +//! OTLP in addition to the console. //! -//! Spans are batch-exported. The batch processor (0.28+) spawns its own -//! background worker and no longer takes a runtime argument; Ring runs on a -//! multi-thread tokio runtime, which sidesteps the current-thread shutdown -//! deadlock documented upstream. +//! The guard ([`OtelGuard`]) owns whichever providers were built and flushes +//! them on drop: dropping without flushing would lose the last, un-exported +//! batch. Keep it alive for the whole process (bind it in `main`), and it shuts +//! every pipeline down cleanly on exit. //! -//! Failure is non-fatal by design: if the exporter can't be built (bad -//! endpoint, etc.) the caller logs and continues logs-only rather than crash. +//! Spans and logs are batch-exported; metrics are pushed on a periodic reader. +//! The batch processor (0.28+) spawns its own background worker and no longer +//! takes a runtime argument; Ring runs on a multi-thread tokio runtime, which +//! sidesteps the current-thread shutdown deadlock documented upstream. +//! +//! Failure is non-fatal by design: if an exporter can't be built (bad endpoint, +//! etc.) the caller logs and continues without that signal rather than crash. use opentelemetry::trace::TracerProvider as _; use opentelemetry_otlp::WithExportConfig as _; use opentelemetry_sdk::Resource; +use opentelemetry_sdk::metrics::{PeriodicReader, SdkMeterProvider}; use opentelemetry_sdk::trace::{Sampler, SdkTracerProvider}; +use std::time::Duration; use tracing::Instrument as _; -use crate::config::server::TracesConfig; +use crate::config::server::{LogsConfig, MetricsConfig, TracesConfig}; +use crate::scheduler::stats_cache::StatsCache; /// Parse the `sampler` config string into an OpenTelemetry [`Sampler`]. /// @@ -56,21 +70,73 @@ fn parse_sampler(spec: &str) -> Sampler { } } -/// Keeps the tracer provider alive for the process lifetime and flushes pending -/// spans on shutdown. Drop it (or call [`OtelGuard::shutdown`]) to force-flush -/// the last batch. Held by `main` so the pipeline lives as long as the server. +/// Keeps whichever OTel providers were built alive for the process lifetime and +/// flushes them on shutdown. Drop it (or call [`OtelGuard::shutdown`]) to +/// force-flush the last batch of each signal. Held by `main` so the pipelines +/// live as long as the server. Each provider is optional: a signal left +/// disabled simply leaves its slot `None`. +#[derive(Default)] pub(crate) struct OtelGuard { - provider: SdkTracerProvider, + tracer: Option, + meter: Option, + logger: Option, } impl OtelGuard { - /// Flush and stop the exporter. Idempotent enough for a shutdown path. + /// Move another guard's tracer provider into this one. Used by `init_tracing` + /// to fold the guard [`build_tracer`] returns into the process-wide guard. + pub(crate) fn absorb(&mut self, other: OtelGuard) { + let mut other = other; + if let Some(p) = other.tracer.take() { + self.tracer = Some(p); + } + if let Some(p) = other.meter.take() { + self.meter = Some(p); + } + if let Some(p) = other.logger.take() { + self.logger = Some(p); + } + // `other` is dropped here, but its providers have been moved out, so its + // `Drop` shuts nothing down. + } + + /// Attach the logger provider so the guard flushes it on shutdown. + pub(crate) fn attach_logger(&mut self, provider: opentelemetry_sdk::logs::SdkLoggerProvider) { + self.logger = Some(provider); + } + + /// Attach the meter provider (built later, once the stats cache exists) so + /// the guard flushes it on shutdown. + pub(crate) fn attach_meter(&mut self, provider: SdkMeterProvider) { + self.meter = Some(provider); + } + + /// Flush and stop every built exporter. Idempotent enough for a shutdown + /// path; a signal that was never enabled is skipped. pub(crate) fn shutdown(&self) { - if let Err(e) = self.provider.force_flush() { - warn!("telemetry: failed to flush spans on shutdown: {e}"); + if let Some(p) = &self.tracer { + if let Err(e) = p.force_flush() { + warn!("telemetry: failed to flush spans on shutdown: {e}"); + } + if let Err(e) = p.shutdown() { + warn!("telemetry: failed to shut tracer provider down: {e}"); + } + } + if let Some(p) = &self.meter { + if let Err(e) = p.force_flush() { + warn!("telemetry: failed to flush metrics on shutdown: {e}"); + } + if let Err(e) = p.shutdown() { + warn!("telemetry: failed to shut meter provider down: {e}"); + } } - if let Err(e) = self.provider.shutdown() { - warn!("telemetry: failed to shut tracer provider down: {e}"); + if let Some(p) = &self.logger { + if let Err(e) = p.force_flush() { + warn!("telemetry: failed to flush logs on shutdown: {e}"); + } + if let Err(e) = p.shutdown() { + warn!("telemetry: failed to shut logger provider down: {e}"); + } } } } @@ -114,7 +180,190 @@ where let tracer = provider.tracer("ring-server"); let layer = tracing_opentelemetry::layer().with_tracer(tracer); - Ok((layer, OtelGuard { provider })) + Ok(( + layer, + OtelGuard { + tracer: Some(provider), + meter: None, + logger: None, + }, + )) +} + +/// Build the OTLP metrics pipeline: an [`SdkMeterProvider`] with a periodic push +/// reader and an observable callback that reads the shared stats cache and +/// reports the per-deployment runtime gauges. Returns the provider (to be held +/// by the [`OtelGuard`]) or `Err` if the exporter can't be built so the caller +/// can continue without metrics rather than crash. +/// +/// No new runtime work is done here: the callback reads the same in-memory +/// `StatsSnapshot` the background stats cache refreshes for the Prometheus +/// `/metrics` endpoint, so enabling OTLP metrics adds no runtime round-trips. +pub(crate) fn build_meter( + cfg: &MetricsConfig, + stats: StatsCache, +) -> anyhow::Result { + let exporter = opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .with_endpoint(cfg.resolved_endpoint()) + .build()?; + + let reader = PeriodicReader::builder(exporter) + .with_interval(Duration::from_secs(cfg.resolved_interval_seconds())) + .build(); + + let resource = Resource::builder() + .with_service_name(cfg.resolved_service_name()) + .build(); + + let provider = SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(resource) + .build(); + + register_deployment_gauges(&provider, stats); + + Ok(provider) +} + +/// Register the per-deployment resource gauges on `provider`. Each is an +/// observable gauge whose callback snapshots the stats cache and emits one +/// measurement per deployment, labelled `{deployment, namespace, runtime}` (the +/// same label set the Prometheus renderer uses). +/// +/// A poisoned cache lock yields no measurements for that collection cycle +/// (logged) rather than panicking the exporter thread. +fn register_deployment_gauges(provider: &SdkMeterProvider, stats: StatsCache) { + use opentelemetry::KeyValue; + use opentelemetry::metrics::MeterProvider as _; + + let meter = provider.meter("ring-server"); + + // One helper per numeric field. Each gauge clones the `Arc` cache handle and + // the field extractor into its callback. `f64` gauges keep a single + // instrument type across integer byte counts and the CPU percentage. + macro_rules! deployment_gauge { + ($name:expr, $unit:expr, $desc:expr, $extract:expr) => {{ + let stats = stats.clone(); + let extract: fn(&crate::scheduler::stats_cache::DeploymentRuntimeStats) -> f64 = + $extract; + meter + .f64_observable_gauge($name) + .with_unit($unit) + .with_description($desc) + .with_callback(move |observer| match stats.read() { + Ok(snap) => { + for d in &snap.deployments { + observer.observe( + extract(d), + &[ + KeyValue::new("deployment", d.name.clone()), + KeyValue::new("namespace", d.namespace.clone()), + KeyValue::new("runtime", d.runtime.clone()), + ], + ); + } + } + Err(e) => { + warn!("telemetry: stats cache lock poisoned in metrics callback: {e}") + } + }) + .build(); + }}; + } + + deployment_gauge!( + "ring.deployment.cpu.percent", + "percent", + "Total CPU usage across the deployment's instances", + |d| d.cpu_usage_percent + ); + deployment_gauge!( + "ring.deployment.memory.used_bytes", + "By", + "Total memory used across the deployment's instances", + |d| d.memory_usage_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.memory.limit_bytes", + "By", + "Total memory limit across the deployment's instances", + |d| d.memory_limit_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.network.rx_bytes", + "By", + "Total network bytes received across the deployment's instances", + |d| d.network_rx_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.network.tx_bytes", + "By", + "Total network bytes transmitted across the deployment's instances", + |d| d.network_tx_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.disk.read_bytes", + "By", + "Total disk bytes read across the deployment's instances", + |d| d.disk_read_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.disk.write_bytes", + "By", + "Total disk bytes written across the deployment's instances", + |d| d.disk_write_bytes as f64 + ); + deployment_gauge!( + "ring.deployment.pids", + "{pid}", + "Total process/thread count across the deployment's instances", + |d| d.pids as f64 + ); + deployment_gauge!( + "ring.deployment.instances", + "{instance}", + "Number of live instances in the deployment", + |d| d.instance_count as f64 + ); + deployment_gauge!( + "ring.deployment.restarts", + "{restart}", + "Total restart count across the deployment's instances", + |d| d.restarts as f64 + ); +} + +/// Build the OTLP logs pipeline: an [`SdkLoggerProvider`] with a batch exporter +/// and an `opentelemetry-appender-tracing` layer that bridges `tracing` events +/// to OTLP. Returns `(layer, provider)` or `Err` if the exporter can't be built +/// so the caller can continue console-only rather than crash. +pub(crate) fn build_logger( + cfg: &LogsConfig, +) -> anyhow::Result<( + opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge< + opentelemetry_sdk::logs::SdkLoggerProvider, + opentelemetry_sdk::logs::SdkLogger, + >, + opentelemetry_sdk::logs::SdkLoggerProvider, +)> { + let exporter = opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .with_endpoint(cfg.resolved_endpoint()) + .build()?; + + let resource = Resource::builder() + .with_service_name(cfg.resolved_service_name()) + .build(); + + let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder() + .with_batch_exporter(exporter) + .with_resource(resource) + .build(); + + let layer = opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&provider); + + Ok((layer, provider)) } /// axum middleware: open one `tracing` span per HTTP request, carrying the From d6a15ce8bd92d019525ea2c9f9becf10b53ffffd Mon Sep 17 00:00:00 2001 From: Mlanawo MBECHEZI Date: Sat, 4 Jul 2026 15:12:47 +0300 Subject: [PATCH 3/3] docs(telemetry): document metrics/logs config and add OTLP e2e tests --- CHANGELOG.md | 2 + documentation/reference/config-toml.md | 29 +++ .../e2e/server/t16_telemetry_metrics_boot.sh | 54 ++++++ tests/e2e/server/t17_telemetry_logs_boot.sh | 49 +++++ .../server/t18_telemetry_otlp_ingestion.sh | 168 ++++++++++++++++++ 5 files changed, 302 insertions(+) create mode 100755 tests/e2e/server/t16_telemetry_metrics_boot.sh create mode 100755 tests/e2e/server/t17_telemetry_logs_boot.sh create mode 100755 tests/e2e/server/t18_telemetry_otlp_ingestion.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f23baa..5f7c8b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - OpenTelemetry distributed tracing (opt-in) via `[server.telemetry.traces]`: `ring server start` exports spans over OTLP/gRPC — one per HTTP request (stable HTTP-server semantic-convention attributes) and one per productive scheduler cycle. Endpoint, `service.name` and sampler are configurable, standard `OTEL_*` env vars override the TOML, and an unreachable collector degrades gracefully instead of failing the server +- OpenTelemetry metric export (opt-in, push) via `[server.telemetry.metrics]`: periodically pushes the per-deployment resource gauges (CPU, memory, network, disk, PIDs, instance and restart counts, labelled `{deployment, namespace, runtime}`) over OTLP/gRPC, read from the same stats cache the Prometheus `/metrics` endpoint uses so no extra runtime round-trip is added. Endpoint, `service.name` and push interval are configurable +- OpenTelemetry log export (opt-in, push) via `[server.telemetry.logs]`: bridges the server's log events to an OTLP/gRPC collector in addition to the console. Each telemetry signal (traces, metrics, logs) is independently toggled; all three degrade gracefully when the collector is unreachable ## [0.10.0] - 2026-07-04 diff --git a/documentation/reference/config-toml.md b/documentation/reference/config-toml.md index 6c54b58..f1bcdbc 100644 --- a/documentation/reference/config-toml.md +++ b/documentation/reference/config-toml.md @@ -16,6 +16,8 @@ A context describes one client→server connection; it has no business deciding [server.scheduler] # optional [server.dashboard] # optional [server.telemetry.traces] # opt-in: enabled = true +[server.telemetry.metrics] # opt-in: enabled = true +[server.telemetry.logs] # opt-in: enabled = true [server.runtime.docker] # opt-in: enabled = true [server.runtime.cloud_hypervisor] # opt-in: enabled = true @@ -91,6 +93,33 @@ Standard `OTEL_*` environment variables take precedence over the TOML values, so Ring emits one span per HTTP request (named `{method} {route}` with the stable HTTP-server semantic-convention attributes) and one `scheduler.cycle` span per reconciliation cycle that has work to do — idle scheduler ticks produce no spans. +Each telemetry signal is independent: you can enable traces, metrics, or logs on their own or in any combination. All three are off by default. + +### `[server.telemetry.metrics]` + +Opt-in OpenTelemetry metric export (push) over OTLP/gRPC. Off by default. When enabled, Ring periodically pushes the same per-deployment resource series the Prometheus `/metrics` endpoint serves, read from the background stats cache (so enabling it adds no extra runtime round-trip). Only `ring server start` exports metrics. + +| Field | Type | Required | Default | Purpose | +|---|---|---|---|---| +| `enabled` | bool | no | `false` | Push metrics to an OTLP/gRPC collector | +| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`) | +| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` | +| `interval_seconds` | int | no | `15` | Push interval. Floored at 1s | + +The exported gauges, labelled `{deployment, namespace, runtime}`, are: `ring.deployment.cpu.percent`, `ring.deployment.memory.used_bytes`, `ring.deployment.memory.limit_bytes`, `ring.deployment.network.rx_bytes`, `ring.deployment.network.tx_bytes`, `ring.deployment.disk.read_bytes`, `ring.deployment.disk.write_bytes`, `ring.deployment.pids`, `ring.deployment.instances`, and `ring.deployment.restarts`. Values are at most one stats-cache refresh (the scheduler interval) stale. A failed exporter is logged and the server continues without OTLP metrics. + +### `[server.telemetry.logs]` + +Opt-in OpenTelemetry log export (push) over OTLP/gRPC. Off by default. When enabled, Ring's log events are bridged to an OTLP collector **in addition to** the console — console logging is never turned off. Only `ring server start` exports logs. + +| Field | Type | Required | Default | Purpose | +|---|---|---|---|---| +| `enabled` | bool | no | `false` | Export log records to an OTLP/gRPC collector | +| `endpoint` | string | no | `"http://127.0.0.1:4317"` | Collector endpoint. Override with `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`) | +| `service_name` | string | no | `"ring-server"` | `service.name` resource attribute. Override with `OTEL_SERVICE_NAME` | + +A failed exporter is logged and the server continues console-only rather than failing. + ### `[server.runtime.docker]` | Field | Type | Required | Default | Purpose | diff --git a/tests/e2e/server/t16_telemetry_metrics_boot.sh b/tests/e2e/server/t16_telemetry_metrics_boot.sh new file mode 100755 index 0000000..fbb73c5 --- /dev/null +++ b/tests/e2e/server/t16_telemetry_metrics_boot.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# T16-server: enabling OTLP metrics export wires up cleanly and degrades +# gracefully when the collector is unreachable. +# +# Autonomous test (own short-lived Ring, no collector). Real metric ingestion +# is covered separately by t18 (which spins a collector in Docker) and by the +# unit tests in `src/config/server.rs`. Here we assert the operational +# contract: +# +# Invariants: +# 1. With `[server.telemetry.metrics] enabled = true` pointing at a closed +# port, Ring still boots and becomes healthy — a metrics export failure +# must never take the server down. +# 2. The startup log confirms metrics export was enabled (so a +# misconfiguration that silently disables it is visible). +# 3. The `/metrics` Prometheus endpoint still serves normally with OTLP +# metrics on (the two share the stats cache and must not interfere). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +source "$SCRIPT_DIR/../lib.sh" + +log "== T16-server: telemetry metrics boot + graceful degradation ==" + +# Point the exporter at a closed port on purpose: the periodic push must fail +# silently in the background, not stop the server from coming up. +export RING_EXTRA_CONFIG='[server.telemetry.metrics] +enabled = true +endpoint = "http://127.0.0.1:14317" +service_name = "ring-e2e" +interval_seconds = 2' + +# Invariant 1: start_ring only returns 0 once /healthz answers, so a successful +# return already proves the server booted with an unreachable collector. +start_ring + +# Invariant 3: the Prometheus /metrics endpoint still answers with OTLP metrics +# enabled (both read the same stats cache). +if ! curl -sf "${RING_URL}/metrics" > /dev/null 2>&1; then + fail "/metrics did not answer with OTLP metrics enabled" +fi +log "server healthy and /metrics serving with OTLP metrics enabled" + +# Invariant 2: the enable log line is present. +if ! grep -q "OTLP metrics export enabled" "$RING_TEST_DIR/ring.log"; then + echo "[e2e] ring.log:" >&2 + cat "$RING_TEST_DIR/ring.log" >&2 + fail "expected 'OTLP metrics export enabled' in the startup log" +fi +log "startup log confirms OTLP metrics export was enabled" + +log "PASS T16-server: telemetry metrics boot" diff --git a/tests/e2e/server/t17_telemetry_logs_boot.sh b/tests/e2e/server/t17_telemetry_logs_boot.sh new file mode 100755 index 0000000..bfc09c7 --- /dev/null +++ b/tests/e2e/server/t17_telemetry_logs_boot.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# T17-server: enabling OTLP log export wires up cleanly and degrades gracefully +# when the collector is unreachable. +# +# Autonomous test (own short-lived Ring, no collector). Real log ingestion is +# covered separately by t18 (which spins a collector in Docker). Here we assert +# the operational contract: +# +# Invariants: +# 1. With `[server.telemetry.logs] enabled = true` pointing at a closed port, +# Ring still boots and becomes healthy — a log export failure must never +# take the server down. +# 2. The startup log confirms log export was enabled. +# 3. Console logging still works with the OTLP log bridge in the subscriber +# stack (the startup banner and enable line are still on stdout/stderr). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +source "$SCRIPT_DIR/../lib.sh" + +log "== T17-server: telemetry logs boot + graceful degradation ==" + +export RING_EXTRA_CONFIG='[server.telemetry.logs] +enabled = true +endpoint = "http://127.0.0.1:14317" +service_name = "ring-e2e"' + +# Invariant 1: booting healthy against a closed collector port proves graceful +# degradation. +start_ring + +# Invariant 3: the API still answers (console + OTLP log layers coexist). +if ! curl -sf "${RING_URL}/healthz" > /dev/null 2>&1; then + fail "healthz did not answer with OTLP logs enabled" +fi +log "server healthy and serving with OTLP logs enabled" + +# Invariant 2: the enable log line is present on the console (proving console +# logging survives adding the OTLP bridge layer). +if ! grep -q "OTLP log export enabled" "$RING_TEST_DIR/ring.log"; then + echo "[e2e] ring.log:" >&2 + cat "$RING_TEST_DIR/ring.log" >&2 + fail "expected 'OTLP log export enabled' in the startup log" +fi +log "startup log confirms OTLP log export was enabled (console logging intact)" + +log "PASS T17-server: telemetry logs boot" diff --git a/tests/e2e/server/t18_telemetry_otlp_ingestion.sh b/tests/e2e/server/t18_telemetry_otlp_ingestion.sh new file mode 100755 index 0000000..5172377 --- /dev/null +++ b/tests/e2e/server/t18_telemetry_otlp_ingestion.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# T18-server: real OTLP ingestion — metrics and logs actually reach a collector. +# +# t16/t17 prove Ring boots and degrades gracefully with an unreachable +# collector; they do NOT prove the exporters emit the right data. This test +# closes that gap end to end: it runs a real OpenTelemetry Collector in Docker +# with a `file` exporter, points Ring at it, deploys a real container so the +# stats cache has data to report, and then asserts that the collector's output +# file contains our metric names and a Ring log record. +# +# Requires Docker (for both the collector and the workload). Skips cleanly when +# Docker is absent, like the other runtime-dependent e2e tests. +# +# Invariants: +# 1. The collector receives at least one of the `ring.deployment.*` metric +# series (proves the metrics push pipeline emits real measurements). +# 2. The collector receives a log record carrying Ring's `service.name` +# (proves the tracing→OTLP log bridge exports real events). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../lib.sh +source "$SCRIPT_DIR/../lib.sh" + +log "== T18-server: real OTLP ingestion (metrics + logs via collector) ==" + +if ! command -v docker > /dev/null 2>&1 || ! docker info > /dev/null 2>&1; then + log "SKIP T18-server: docker not available" + exit 0 +fi + +COLLECTOR_IMAGE="otel/opentelemetry-collector-contrib:0.115.1" +COLLECTOR_NAME="ring-e2e-otelcol-$$" +COLLECTOR_PORT=14318 +OTEL_DIR="$(mktemp -d -t ring-otel-XXXXXX)" + +# Collector config: receive OTLP/gRPC, write everything to a file we can read. +# The output dir is mounted from the host so the test reads the file directly. +cat > "$OTEL_DIR/config.yaml" <<'YAML' +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +exporters: + file: + path: /out/telemetry.json +service: + pipelines: + metrics: + receivers: [otlp] + exporters: [file] + logs: + receivers: [otlp] + exporters: [file] +YAML + +# The collector image runs as a non-root user (uid 10001); it must be able to +# create its output file in the mounted dir, so make the dir world-writable and +# let the collector create the file itself (a host-created file would be owned +# by our uid and rejected with EACCES). +chmod 777 "$OTEL_DIR" + +cleanup_collector() { + docker rm -f "$COLLECTOR_NAME" > /dev/null 2>&1 || true + rm -rf "$OTEL_DIR" 2>/dev/null || true +} +trap cleanup_collector EXIT + +log "starting OTLP collector ($COLLECTOR_IMAGE) on 127.0.0.1:${COLLECTOR_PORT}" +docker run -d --name "$COLLECTOR_NAME" \ + -p "127.0.0.1:${COLLECTOR_PORT}:4317" \ + -v "$OTEL_DIR/config.yaml:/etc/otelcol-contrib/config.yaml:ro" \ + -v "$OTEL_DIR:/out" \ + "$COLLECTOR_IMAGE" > /dev/null + +# Wait for the collector to stay up and its gRPC port to accept a TCP connection. +# The image is distroless (no shell), so probe from the host, not via docker exec. +collector_up="" +for _ in $(seq 1 30); do + state=$(docker inspect -f '{{.State.Running}}' "$COLLECTOR_NAME" 2>/dev/null || echo false) + if [ "$state" != "true" ]; then + echo "[e2e] collector container is not running:" >&2 + docker logs "$COLLECTOR_NAME" 2>&1 | tail -20 >&2 || true + fail "OTLP collector failed to start" + fi + # /dev/tcp is a bash builtin: a successful open proves the port is listening. + if (exec 3<>"/dev/tcp/127.0.0.1/${COLLECTOR_PORT}") 2>/dev/null; then + collector_up=1 + exec 3>&- 2>/dev/null || true + break + fi + sleep 0.5 +done +if [ -z "$collector_up" ]; then + docker logs "$COLLECTOR_NAME" 2>&1 | tail -20 >&2 || true + fail "OTLP collector gRPC port ${COLLECTOR_PORT} never accepted connections" +fi +log "collector is up and accepting connections" + +# Enable both signals, short push interval so metrics arrive quickly. +export RING_EXTRA_CONFIG="[server.telemetry.metrics] +enabled = true +endpoint = \"http://127.0.0.1:${COLLECTOR_PORT}\" +service_name = \"ring-e2e-otlp\" +interval_seconds = 2 + +[server.telemetry.logs] +enabled = true +endpoint = \"http://127.0.0.1:${COLLECTOR_PORT}\" +service_name = \"ring-e2e-otlp\"" + +start_ring +ring_login + +# Deploy a real container so the stats cache has a running deployment to report. +NS="ring-e2e" +NAME="otel-metrics-target" +cat > "$RING_TEST_DIR/deploy.yaml" < /dev/null +wait_deployment_status "$NS" "$NAME" "running" 60 + +# Give the stats cache a refresh cycle and the metrics reader a push interval, +# plus the log batch a flush. +log "waiting for the collector to receive telemetry..." +OUT="$OTEL_DIR/telemetry.json" +got_metric="" +got_log="" +for _ in $(seq 1 40); do + if [ -z "$got_metric" ] && grep -q "ring.deployment." "$OUT" 2>/dev/null; then + got_metric=1 + fi + if [ -z "$got_log" ] && grep -q "ring-e2e-otlp" "$OUT" 2>/dev/null; then + got_log=1 + fi + if [ -n "$got_metric" ] && [ -n "$got_log" ]; then + break + fi + sleep 1 +done + +# Invariant 1: a ring.deployment.* metric series reached the collector. +if [ -z "$got_metric" ]; then + echo "[e2e] collector output (no ring.deployment.* metric found):" >&2 + tail -c 4000 "$OUT" >&2 || true + fail "expected a 'ring.deployment.*' metric in the collector output" +fi +log "collector received ring.deployment.* metrics" + +# Invariant 2: a Ring log record reached the collector (service.name present). +if [ -z "$got_log" ]; then + echo "[e2e] collector output (no ring-e2e-otlp log record found):" >&2 + tail -c 4000 "$OUT" >&2 || true + fail "expected a Ring log record (service.name=ring-e2e-otlp) in the collector output" +fi +log "collector received Ring log records" + +log "PASS T18-server: real OTLP ingestion"