From 03a78a3d6cc7ef2c0e220b1bc16acb12d6dffa8c Mon Sep 17 00:00:00 2001 From: evalir Date: Thu, 7 May 2026 17:15:06 +0200 Subject: [PATCH 1/3] feat: add host_blocks_seen counter and last_host_block_seen_timestamp gauge Records two metrics on the ingress side of the EnvTask rollup-block subscription, labeled by host_chain_id. They advance even when the builder skips block construction, so operators can detect a silently-dead WS subscription that previously looked identical to a builder choosing not to build. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 39 ++++++++++++++++++ Cargo.toml | 1 + src/metrics.rs | 104 ++++++++++++++++++++++++++++++++++++++++++++++- src/tasks/env.rs | 2 + 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d73e934..36f0d9eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2381,6 +2381,7 @@ dependencies = [ "git-version", "init4-bin-base", "itertools 0.14.0", + "metrics-util", "openssl", "reqwest", "reth-chainspec", @@ -3608,6 +3609,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "enr" version = "0.13.0" @@ -5868,11 +5875,15 @@ version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e56997f084e57b045edf17c3ed8ba7f9f779c670df8206dfd1c736f4c02dc4a" dependencies = [ + "aho-corasick", "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.16.1", + "indexmap 2.14.0", "metrics", + "ordered-float", "quanta", + "radix_trie", "rand 0.9.4", "rand_xoshiro", "rapidhash", @@ -6024,6 +6035,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nom" version = "7.1.3" @@ -6627,6 +6647,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "outref" version = "0.5.2" @@ -7224,6 +7253,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.6" diff --git a/Cargo.toml b/Cargo.toml index 7d9b2099..df310ea5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ url = "2.5.4" alloy-hardforks = "0.4.0" alloy-chains = "0.2" criterion = { version = "0.8.2", features = ["async_tokio"] } +metrics-util = "0.20" signet-bundle = "0.16.0-rc.11" [[bench]] diff --git a/src/metrics.rs b/src/metrics.rs index bf96c4c4..27bcbf08 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -4,13 +4,30 @@ //! use bare `counter!` / `histogram!` macros directly. This prevents //! metric-name typos and provides a single place to survey every metric the //! builder emits. -use init4_bin_base::deps::metrics::{counter, describe_counter, describe_histogram, histogram}; -use std::sync::LazyLock; +use init4_bin_base::deps::metrics::{ + counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, +}; +use std::{ + sync::LazyLock, + time::{SystemTime, UNIX_EPOCH}, +}; // --------------------------------------------------------------------------- // Metric names and help text // --------------------------------------------------------------------------- +// -- Chain ingress -- + +const HOST_BLOCKS_SEEN: &str = "signet.builder.host_blocks_seen"; +const HOST_BLOCKS_SEEN_HELP: &str = + "Host-chain blocks observed by the builder. Advances even when no block is built."; + +const LAST_HOST_BLOCK_SEEN_TIMESTAMP: &str = "signet.builder.last_host_block_seen_timestamp"; +const LAST_HOST_BLOCK_SEEN_TIMESTAMP_HELP: &str = + "Unix seconds (wall clock) at which the builder most recently observed a host block."; + +const HOST_CHAIN_ID_LABEL: &str = "host_chain_id"; + // -- Block building -- const BUILT_BLOCKS: &str = "signet.builder.built_blocks"; @@ -145,6 +162,10 @@ const PYLON_SIDECARS_SUBMITTED_HELP: &str = "Successful Pylon sidecar submission // --------------------------------------------------------------------------- static DESCRIPTIONS: LazyLock<()> = LazyLock::new(|| { + // Chain ingress + describe_counter!(HOST_BLOCKS_SEEN, HOST_BLOCKS_SEEN_HELP); + describe_gauge!(LAST_HOST_BLOCK_SEEN_TIMESTAMP, LAST_HOST_BLOCK_SEEN_TIMESTAMP_HELP); + // Block building describe_counter!(BUILT_BLOCKS, BUILT_BLOCKS_HELP); describe_histogram!(BUILT_BLOCKS_TX_COUNT, BUILT_BLOCKS_TX_COUNT_HELP); @@ -204,6 +225,22 @@ pub(crate) fn init() { LazyLock::force(&DESCRIPTIONS); } +// --------------------------------------------------------------------------- +// Public API -- Chain ingress +// --------------------------------------------------------------------------- + +/// Record that the builder has observed a new host-chain block, labeled by +/// `host_chain_id`. +pub(crate) fn record_host_block_seen(host_chain_id: u64) { + counter!(HOST_BLOCKS_SEEN, HOST_CHAIN_ID_LABEL => host_chain_id.to_string()).increment(1); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before UNIX_EPOCH") + .as_secs_f64(); + gauge!(LAST_HOST_BLOCK_SEEN_TIMESTAMP, HOST_CHAIN_ID_LABEL => host_chain_id.to_string()) + .set(now); +} + // --------------------------------------------------------------------------- // Public API -- Block building // --------------------------------------------------------------------------- @@ -421,3 +458,66 @@ pub(crate) fn inc_pylon_submission_failures() { pub(crate) fn inc_pylon_sidecars_submitted() { counter!(PYLON_SIDECARS_SUBMITTED).increment(1); } + +#[cfg(test)] +mod tests { + use super::{ + HOST_BLOCKS_SEEN, HOST_CHAIN_ID_LABEL, LAST_HOST_BLOCK_SEEN_TIMESTAMP, + record_host_block_seen, + }; + use init4_bin_base::deps::metrics::{Label, with_local_recorder}; + use metrics_util::{ + MetricKind, + debugging::{DebugValue, DebuggingRecorder}, + }; + use std::time::{SystemTime, UNIX_EPOCH}; + + /// Verify that each call to `record_host_block_seen` advances the + /// `host_blocks_seen` counter by one and sets the + /// `last_host_block_seen_timestamp` gauge to (approximately) the current + /// wall-clock Unix time, with the chain id attached as a label. + #[test] + fn record_host_block_seen_advances_counter_and_gauge() { + const CHAIN_ID: u64 = 17_001; + const OBSERVATIONS: u64 = 3; + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + + let before = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs_f64(); + with_local_recorder(&recorder, || { + for _ in 0..OBSERVATIONS { + record_host_block_seen(CHAIN_ID); + } + }); + let after = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs_f64(); + + let expected_label = Label::new(HOST_CHAIN_ID_LABEL, CHAIN_ID.to_string()); + + let mut counter_value = None; + let mut gauge_value = None; + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + let labels: Vec<_> = key.key().labels().cloned().collect(); + assert!( + labels.contains(&expected_label), + "metric {} missing host_chain_id label, found: {labels:?}", + key.key().name() + ); + match (key.kind(), key.key().name()) { + (MetricKind::Counter, HOST_BLOCKS_SEEN) => counter_value = Some(value), + (MetricKind::Gauge, LAST_HOST_BLOCK_SEEN_TIMESTAMP) => gauge_value = Some(value), + _ => {} + } + } + + assert_eq!(counter_value, Some(DebugValue::Counter(OBSERVATIONS))); + let Some(DebugValue::Gauge(ts)) = gauge_value else { + panic!("expected gauge value, got {gauge_value:?}"); + }; + let ts: f64 = ts.into(); + assert!( + ts >= before && ts <= after, + "gauge timestamp {ts} not in observed window [{before}, {after}]" + ); + } +} diff --git a/src/tasks/env.rs b/src/tasks/env.rs index f1229225..2fb85fbb 100644 --- a/src/tasks/env.rs +++ b/src/tasks/env.rs @@ -280,6 +280,8 @@ impl EnvTask { drop(span); while let Some(rollup_header) = rollup_headers.next().await { + crate::metrics::record_host_block_seen(self.config.constants.host_chain_id()); + let host_block_number = self.config.constants.rollup_block_to_host_block_num(rollup_header.number); let rollup_block_number = rollup_header.number; From 3e371974a09a52dc4962bbafc861f9638e275130 Mon Sep 17 00:00:00 2001 From: evalir Date: Thu, 7 May 2026 17:21:58 +0200 Subject: [PATCH 2/3] refactor: rename host_blocks_seen to rollup_blocks_seen The chain-following subscription is to rollup blocks (ru_provider.subscribe_blocks), not host blocks. Rename the metric, label, and helper to match what is actually observed; the WS that can silently die is the rollup-block one. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/metrics.rs | 50 ++++++++++++++++++++++++------------------------ src/tasks/env.rs | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index 27bcbf08..8a683cfe 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -18,15 +18,15 @@ use std::{ // -- Chain ingress -- -const HOST_BLOCKS_SEEN: &str = "signet.builder.host_blocks_seen"; -const HOST_BLOCKS_SEEN_HELP: &str = - "Host-chain blocks observed by the builder. Advances even when no block is built."; +const ROLLUP_BLOCKS_SEEN: &str = "signet.builder.rollup_blocks_seen"; +const ROLLUP_BLOCKS_SEEN_HELP: &str = + "Rollup-chain blocks observed by the builder. Advances even when no block is built."; -const LAST_HOST_BLOCK_SEEN_TIMESTAMP: &str = "signet.builder.last_host_block_seen_timestamp"; -const LAST_HOST_BLOCK_SEEN_TIMESTAMP_HELP: &str = - "Unix seconds (wall clock) at which the builder most recently observed a host block."; +const LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP: &str = "signet.builder.last_rollup_block_seen_timestamp"; +const LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP_HELP: &str = + "Unix seconds (wall clock) at which the builder most recently observed a rollup block."; -const HOST_CHAIN_ID_LABEL: &str = "host_chain_id"; +const ROLLUP_CHAIN_ID_LABEL: &str = "rollup_chain_id"; // -- Block building -- @@ -163,8 +163,8 @@ const PYLON_SIDECARS_SUBMITTED_HELP: &str = "Successful Pylon sidecar submission static DESCRIPTIONS: LazyLock<()> = LazyLock::new(|| { // Chain ingress - describe_counter!(HOST_BLOCKS_SEEN, HOST_BLOCKS_SEEN_HELP); - describe_gauge!(LAST_HOST_BLOCK_SEEN_TIMESTAMP, LAST_HOST_BLOCK_SEEN_TIMESTAMP_HELP); + describe_counter!(ROLLUP_BLOCKS_SEEN, ROLLUP_BLOCKS_SEEN_HELP); + describe_gauge!(LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP, LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP_HELP); // Block building describe_counter!(BUILT_BLOCKS, BUILT_BLOCKS_HELP); @@ -229,15 +229,15 @@ pub(crate) fn init() { // Public API -- Chain ingress // --------------------------------------------------------------------------- -/// Record that the builder has observed a new host-chain block, labeled by -/// `host_chain_id`. -pub(crate) fn record_host_block_seen(host_chain_id: u64) { - counter!(HOST_BLOCKS_SEEN, HOST_CHAIN_ID_LABEL => host_chain_id.to_string()).increment(1); +/// Record that the builder has observed a new rollup-chain block, labeled by +/// `rollup_chain_id`. +pub(crate) fn record_rollup_block_seen(rollup_chain_id: u64) { + counter!(ROLLUP_BLOCKS_SEEN, ROLLUP_CHAIN_ID_LABEL => rollup_chain_id.to_string()).increment(1); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("system clock before UNIX_EPOCH") .as_secs_f64(); - gauge!(LAST_HOST_BLOCK_SEEN_TIMESTAMP, HOST_CHAIN_ID_LABEL => host_chain_id.to_string()) + gauge!(LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP, ROLLUP_CHAIN_ID_LABEL => rollup_chain_id.to_string()) .set(now); } @@ -462,8 +462,8 @@ pub(crate) fn inc_pylon_sidecars_submitted() { #[cfg(test)] mod tests { use super::{ - HOST_BLOCKS_SEEN, HOST_CHAIN_ID_LABEL, LAST_HOST_BLOCK_SEEN_TIMESTAMP, - record_host_block_seen, + LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP, ROLLUP_BLOCKS_SEEN, ROLLUP_CHAIN_ID_LABEL, + record_rollup_block_seen, }; use init4_bin_base::deps::metrics::{Label, with_local_recorder}; use metrics_util::{ @@ -472,12 +472,12 @@ mod tests { }; use std::time::{SystemTime, UNIX_EPOCH}; - /// Verify that each call to `record_host_block_seen` advances the - /// `host_blocks_seen` counter by one and sets the - /// `last_host_block_seen_timestamp` gauge to (approximately) the current + /// Verify that each call to `record_rollup_block_seen` advances the + /// `rollup_blocks_seen` counter by one and sets the + /// `last_rollup_block_seen_timestamp` gauge to (approximately) the current /// wall-clock Unix time, with the chain id attached as a label. #[test] - fn record_host_block_seen_advances_counter_and_gauge() { + fn record_rollup_block_seen_advances_counter_and_gauge() { const CHAIN_ID: u64 = 17_001; const OBSERVATIONS: u64 = 3; @@ -487,12 +487,12 @@ mod tests { let before = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs_f64(); with_local_recorder(&recorder, || { for _ in 0..OBSERVATIONS { - record_host_block_seen(CHAIN_ID); + record_rollup_block_seen(CHAIN_ID); } }); let after = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs_f64(); - let expected_label = Label::new(HOST_CHAIN_ID_LABEL, CHAIN_ID.to_string()); + let expected_label = Label::new(ROLLUP_CHAIN_ID_LABEL, CHAIN_ID.to_string()); let mut counter_value = None; let mut gauge_value = None; @@ -500,12 +500,12 @@ mod tests { let labels: Vec<_> = key.key().labels().cloned().collect(); assert!( labels.contains(&expected_label), - "metric {} missing host_chain_id label, found: {labels:?}", + "metric {} missing rollup_chain_id label, found: {labels:?}", key.key().name() ); match (key.kind(), key.key().name()) { - (MetricKind::Counter, HOST_BLOCKS_SEEN) => counter_value = Some(value), - (MetricKind::Gauge, LAST_HOST_BLOCK_SEEN_TIMESTAMP) => gauge_value = Some(value), + (MetricKind::Counter, ROLLUP_BLOCKS_SEEN) => counter_value = Some(value), + (MetricKind::Gauge, LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP) => gauge_value = Some(value), _ => {} } } diff --git a/src/tasks/env.rs b/src/tasks/env.rs index 2fb85fbb..1b86e1f5 100644 --- a/src/tasks/env.rs +++ b/src/tasks/env.rs @@ -280,7 +280,7 @@ impl EnvTask { drop(span); while let Some(rollup_header) = rollup_headers.next().await { - crate::metrics::record_host_block_seen(self.config.constants.host_chain_id()); + crate::metrics::record_rollup_block_seen(self.config.constants.ru_chain_id()); let host_block_number = self.config.constants.rollup_block_to_host_block_num(rollup_header.number); From 3039337ac9b0b0e10653b10744f4ba1edcc6223d Mon Sep 17 00:00:00 2001 From: evalir Date: Fri, 8 May 2026 14:13:38 +0200 Subject: [PATCH 3/3] test: panic on unexpected metric in test scope Address review nit: the local recorder in record_rollup_block_seen_advances_counter_and_gauge should only ever see the two metrics under test. Silently no-op'ing on anything else hides isolation bugs. Bind the catch-all so the panic names what leaked in. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics.rs b/src/metrics.rs index 8a683cfe..7f2a917a 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -506,7 +506,7 @@ mod tests { match (key.kind(), key.key().name()) { (MetricKind::Counter, ROLLUP_BLOCKS_SEEN) => counter_value = Some(value), (MetricKind::Gauge, LAST_ROLLUP_BLOCK_SEEN_TIMESTAMP) => gauge_value = Some(value), - _ => {} + (kind, name) => panic!("unexpected metric in test scope: {name} ({kind:?})"), } }