From f8096f683654bc9be3eeea8846ac47b3cea97104 Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:21:27 -0700 Subject: [PATCH 1/8] check l2 client health in background worker first --- Cargo.lock | 41 ++++++ crates/rollup-boost/Cargo.toml | 1 + crates/rollup-boost/src/health.rs | 213 +++++++++++++++++++++++++----- crates/rollup-boost/src/server.rs | 1 + 4 files changed, 221 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83728d95..c6a1d88f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10034,6 +10034,7 @@ dependencies = [ "rustls", "serde", "serde_json", + "serial_test", "sha2 0.10.9", "testcontainers", "thiserror 2.0.12", @@ -10293,6 +10294,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.27" @@ -10349,6 +10359,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "sec1" version = "0.7.3" @@ -10597,6 +10613,31 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/crates/rollup-boost/Cargo.toml b/crates/rollup-boost/Cargo.toml index 5f1bf069..58378628 100644 --- a/crates/rollup-boost/Cargo.toml +++ b/crates/rollup-boost/Cargo.toml @@ -66,6 +66,7 @@ parking_lot = "0.12.3" tokio-util = { version = "0.7.13" } [dev-dependencies] +serial_test = "*" rand = "0.9.0" time = { version = "0.3.36", features = ["macros", "formatting", "parsing"] } op-alloy-consensus = "0.17.2" diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index fed2e27c..0d9130b9 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -9,13 +9,14 @@ use tokio::{ task::JoinHandle, time::{Instant, sleep_until}, }; -use tracing::warn; +use tracing::{info, warn}; use crate::{EngineApiExt, ExecutionMode, Health, Probes}; pub struct HealthHandle { pub probes: Arc, pub execution_mode: Arc>, + pub l2_client: Arc, pub builder_client: Arc, pub health_check_interval: Duration, pub max_unsafe_interval: u64, @@ -26,6 +27,7 @@ impl HealthHandle { pub fn new( probes: Arc, execution_mode: Arc>, + l2_client: Arc, builder_client: Arc, health_check_interval: Duration, max_unsafe_interval: u64, @@ -33,6 +35,7 @@ impl HealthHandle { Self { probes, execution_mode, + l2_client, builder_client, health_check_interval, max_unsafe_interval, @@ -46,35 +49,65 @@ impl HealthHandle { let mut timestamp = MonotonicTimestamp::new(); loop { - let latest_unsafe = match self - .builder_client + sleep_until(Instant::now() + self.health_check_interval).await; + info!("health check loop"); + + let t = timestamp.tick(); + + // Check L2 client health. If its unhealthy, set the health status to ServiceUnavailable + // If in disabled or dry run execution mode, set the health status to Healthy if the l2 client is healthy + match self + .l2_client .get_block_by_number(BlockNumberOrTag::Latest, false) .await { - Ok(block) => block, - Err(e) => { - warn!(target: "rollup_boost::health", "Failed to get unsafe block from builder client: {} - updating health status", e); - if self.execution_mode.lock().is_enabled() { - self.probes.set_health(Health::PartialContent); + Ok(block) => { + if t.saturating_sub(block.header.timestamp) + .gt(&self.max_unsafe_interval) + { + warn!(target: "rollup_boost::health", curr_unix = %t, unsafe_unix = %block.header.timestamp, "L2 client - unsafe block timestamp is too old, updating health status to ServiceUnavailable"); + self.probes.set_health(Health::ServiceUnavailable); + continue; + } else if self.execution_mode.lock().is_disabled() + || self.execution_mode.lock().is_dry_run() + { + info!("self.probes.set_health"); + self.probes.set_health(Health::Healthy); + continue; } - sleep_until(Instant::now() + self.health_check_interval).await; + } + Err(e) => { + warn!(target: "rollup_boost::health", "Failed to get unsafe block from l2 client: {} - updating health status", e); + self.probes.set_health(Health::ServiceUnavailable); continue; } }; - let t = timestamp.tick(); - if t.saturating_sub(latest_unsafe.header.timestamp) - .gt(&self.max_unsafe_interval) - { - warn!(target: "rollup_boost::health", curr_unix = %t, unsafe_unix = %latest_unsafe.header.timestamp, "Unsafe block timestamp is too old updating health status"); - if self.execution_mode.lock().is_enabled() { - self.probes.set_health(Health::PartialContent); - } - } else { - self.probes.set_health(Health::Healthy); + if self.execution_mode.lock().is_enabled() { + info!("self.execution_mode.lock().is_enabled"); + // Only check builder client health if execution mode is enabled + // If its unhealthy, set the health status to PartialContent + match self + .builder_client + .get_block_by_number(BlockNumberOrTag::Latest, false) + .await + { + Ok(block) => { + if t.saturating_sub(block.header.timestamp) + .gt(&self.max_unsafe_interval) + { + warn!(target: "rollup_boost::health", curr_unix = %t, unsafe_unix = %block.header.timestamp, "Builder client - unsafe block timestamp is too old updating health status"); + self.probes.set_health(Health::PartialContent); + } else { + self.probes.set_health(Health::Healthy); + } + } + Err(e) => { + warn!(target: "rollup_boost::health", "Failed to get unsafe block from builder client: {} - updating health status", e); + self.probes.set_health(Health::PartialContent); + } + }; } - - sleep_until(Instant::now() + self.health_check_interval).await; } }) } @@ -141,6 +174,7 @@ mod tests { use super::*; use crate::{Probes, payload::PayloadSource}; + use serial_test::serial; pub struct MockHttpServer { addr: SocketAddr, @@ -258,15 +292,35 @@ mod tests { Ok(hyper::Response::new(response.to_string())) } + async fn error_handler( + _req: hyper::Request, + _block_timestamp: u64, + ) -> Result, hyper::Error> { + let error_response = json!({ + "jsonrpc": "2.0", + "error": { "code": -32603, "message": "Injected test error" }, + "id": 1 + }); + Ok(hyper::Response::new(error_response.to_string())) + } + + #[serial] #[tokio::test] async fn test_health_check_healthy() -> eyre::Result<()> { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; let probes = Arc::new(Probes::default()); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); + let l2 = MockHttpServer::serve(handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + let builder = MockHttpServer::serve(handler, now).await.unwrap(); let builder_client = Arc::new(RpcClient::new( format!("http://{}", builder.addr).parse::()?, @@ -278,8 +332,9 @@ mod tests { let health_handle = HealthHandle { probes: probes.clone(), execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), + l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(60), + health_check_interval: Duration::from_secs(1), max_unsafe_interval: 5, }; @@ -289,16 +344,26 @@ mod tests { Ok(()) } + #[serial] #[tokio::test] - async fn test_health_check_exceeds_max_unsafe_interval() -> eyre::Result<()> { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + async fn test_health_check_builder_exceeds_max_unsafe_interval() -> eyre::Result<()> { let probes = Arc::new(Probes::default()); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); - let builder = MockHttpServer::serve(handler, now - 10).await.unwrap(); + // L2 healthy + let l2 = MockHttpServer::serve(handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + + // Builder unhealthy + let builder = MockHttpServer::serve(handler, now - 10).await.unwrap(); let builder_client = Arc::new(RpcClient::new( format!("http://{}", builder.addr).parse::()?, JwtSecret::random(), @@ -309,8 +374,9 @@ mod tests { let health_handle = HealthHandle { probes: probes.clone(), execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), + l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(60), + health_check_interval: Duration::from_secs(1), max_unsafe_interval: 5, }; @@ -320,15 +386,23 @@ mod tests { Ok(()) } + #[serial] #[tokio::test] async fn test_health_check_exceeds_max_unsafe_interval_execution_mode_disabled() -> eyre::Result<()> { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; let probes = Arc::new(Probes::default()); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); + // L2 healthy + let l2 = MockHttpServer::serve(handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); let builder = MockHttpServer::serve(handler, now - 10).await.unwrap(); let builder_client = Arc::new(RpcClient::new( @@ -341,8 +415,9 @@ mod tests { let health_handle = HealthHandle { probes: probes.clone(), execution_mode: Arc::new(Mutex::new(ExecutionMode::Disabled)), + l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(60), + health_check_interval: Duration::from_secs(1), max_unsafe_interval: 5, }; @@ -352,15 +427,23 @@ mod tests { Ok(()) } + #[serial] #[tokio::test] async fn test_health_check_exceeds_max_unsafe_interval_execution_mode_dryrun() -> eyre::Result<()> { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; let probes = Arc::new(Probes::default()); let now = SystemTime::now() .duration_since(UNIX_EPOCH) .expect("Time went backwards") .as_secs(); + // L2 healthy + let l2 = MockHttpServer::serve(handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); let builder = MockHttpServer::serve(handler, now - 10).await.unwrap(); let builder_client = Arc::new(RpcClient::new( @@ -373,8 +456,9 @@ mod tests { let health_handle = HealthHandle { probes: probes.clone(), execution_mode: Arc::new(Mutex::new(ExecutionMode::DryRun)), + l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(60), + health_check_interval: Duration::from_secs(1), max_unsafe_interval: 5, }; @@ -384,10 +468,24 @@ mod tests { Ok(()) } + #[serial] #[tokio::test] - async fn test_health_check_service_unavailable() -> eyre::Result<()> { - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + async fn test_health_check_service_builder_unavailable() -> eyre::Result<()> { let probes = Arc::new(Probes::default()); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + // L2 healthy + let l2 = MockHttpServer::serve(handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + + // Builder unhealthy let builder_client = Arc::new(RpcClient::new( "http://127.0.0.1:6000".parse::()?, JwtSecret::random(), @@ -398,8 +496,9 @@ mod tests { let health_handle = HealthHandle { probes: probes.clone(), execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), + l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(60), + health_check_interval: Duration::from_secs(1), max_unsafe_interval: 5, }; @@ -409,9 +508,52 @@ mod tests { Ok(()) } + #[serial] + #[tokio::test] + async fn test_health_check_service_l2_unavailable() -> eyre::Result<()> { + let probes = Arc::new(Probes::default()); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + + // L2 returns an error + let l2 = MockHttpServer::serve(error_handler, now).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + + // Builder healthy + let builder = MockHttpServer::serve(handler, now).await.unwrap(); + let builder_client = Arc::new(RpcClient::new( + format!("http://{}", builder.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::Builder, + )?); + + let health_handle = HealthHandle { + probes: probes.clone(), + execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), + l2_client: l2_client.clone(), + builder_client: builder_client.clone(), + health_check_interval: Duration::from_secs(1), + max_unsafe_interval: 5, + }; + + health_handle.spawn(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(matches!(probes.health(), Health::ServiceUnavailable)); + Ok(()) + } + + #[serial] #[tokio::test] async fn tick_advances_after_sleep() { - let mut ts = MonotonicTimestamp::new(); + let mut ts: MonotonicTimestamp = MonotonicTimestamp::new(); let t1 = ts.tick(); tokio::time::sleep(Duration::from_secs(1)).await; let t2 = ts.tick(); @@ -419,6 +561,7 @@ mod tests { assert!(t2 >= t1 + 1,); } + #[serial] #[tokio::test] async fn tick_matches_system_clock() { let mut ts = MonotonicTimestamp::new(); diff --git a/crates/rollup-boost/src/server.rs b/crates/rollup-boost/src/server.rs index d6f315a2..60472559 100644 --- a/crates/rollup-boost/src/server.rs +++ b/crates/rollup-boost/src/server.rs @@ -82,6 +82,7 @@ where let handle = HealthHandle::new( self.probes.clone(), self.execution_mode.clone(), + self.l2_client.clone(), self.builder_client.clone(), Duration::from_secs(health_check_interval), max_unsafe_interval, From ea91d899488f4aa074266124050e42931e5675ef Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:22:42 -0700 Subject: [PATCH 2/8] fix --- crates/rollup-boost/src/health.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index 0d9130b9..84f92053 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -50,10 +50,8 @@ impl HealthHandle { loop { sleep_until(Instant::now() + self.health_check_interval).await; - info!("health check loop"); - let t = timestamp.tick(); - + // Check L2 client health. If its unhealthy, set the health status to ServiceUnavailable // If in disabled or dry run execution mode, set the health status to Healthy if the l2 client is healthy match self From 26c896499a4c04361acf4d60cdcec320f675fe80 Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:25:53 -0700 Subject: [PATCH 3/8] removed unused --- crates/rollup-boost/src/health.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index 84f92053..b7c6a8f6 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -51,7 +51,7 @@ impl HealthHandle { loop { sleep_until(Instant::now() + self.health_check_interval).await; let t = timestamp.tick(); - + // Check L2 client health. If its unhealthy, set the health status to ServiceUnavailable // If in disabled or dry run execution mode, set the health status to Healthy if the l2 client is healthy match self @@ -290,18 +290,6 @@ mod tests { Ok(hyper::Response::new(response.to_string())) } - async fn error_handler( - _req: hyper::Request, - _block_timestamp: u64, - ) -> Result, hyper::Error> { - let error_response = json!({ - "jsonrpc": "2.0", - "error": { "code": -32603, "message": "Injected test error" }, - "id": 1 - }); - Ok(hyper::Response::new(error_response.to_string())) - } - #[serial] #[tokio::test] async fn test_health_check_healthy() -> eyre::Result<()> { From 7d6616b76dc7b31c72b3a08da9b0d657e092c470 Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:26:39 -0700 Subject: [PATCH 4/8] logging --- crates/rollup-boost/src/health.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index b7c6a8f6..7cb45e18 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -75,7 +75,7 @@ impl HealthHandle { } } Err(e) => { - warn!(target: "rollup_boost::health", "Failed to get unsafe block from l2 client: {} - updating health status", e); + warn!(target: "rollup_boost::health", "L2 client - Failed to get unsafe block {} - updating health status", e); self.probes.set_health(Health::ServiceUnavailable); continue; } @@ -101,7 +101,7 @@ impl HealthHandle { } } Err(e) => { - warn!(target: "rollup_boost::health", "Failed to get unsafe block from builder client: {} - updating health status", e); + warn!(target: "rollup_boost::health", "Builder client - Failed to get unsafe block {} - updating health status", e); self.probes.set_health(Health::PartialContent); } }; From 11bfd8728b4e449598d422347b42c515b6acf0f4 Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:28:29 -0700 Subject: [PATCH 5/8] put sleep at end --- crates/rollup-boost/src/health.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index 7cb45e18..730dc7ce 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -49,7 +49,6 @@ impl HealthHandle { let mut timestamp = MonotonicTimestamp::new(); loop { - sleep_until(Instant::now() + self.health_check_interval).await; let t = timestamp.tick(); // Check L2 client health. If its unhealthy, set the health status to ServiceUnavailable @@ -65,24 +64,25 @@ impl HealthHandle { { warn!(target: "rollup_boost::health", curr_unix = %t, unsafe_unix = %block.header.timestamp, "L2 client - unsafe block timestamp is too old, updating health status to ServiceUnavailable"); self.probes.set_health(Health::ServiceUnavailable); + sleep_until(Instant::now() + self.health_check_interval).await; continue; } else if self.execution_mode.lock().is_disabled() || self.execution_mode.lock().is_dry_run() { - info!("self.probes.set_health"); self.probes.set_health(Health::Healthy); + sleep_until(Instant::now() + self.health_check_interval).await; continue; } } Err(e) => { warn!(target: "rollup_boost::health", "L2 client - Failed to get unsafe block {} - updating health status", e); self.probes.set_health(Health::ServiceUnavailable); + sleep_until(Instant::now() + self.health_check_interval).await; continue; } }; if self.execution_mode.lock().is_enabled() { - info!("self.execution_mode.lock().is_enabled"); // Only check builder client health if execution mode is enabled // If its unhealthy, set the health status to PartialContent match self @@ -106,6 +106,7 @@ impl HealthHandle { } }; } + sleep_until(Instant::now() + self.health_check_interval).await; } }) } From d7995e4c39e18d844513e1aff1038a4970aacefd Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:29:17 -0700 Subject: [PATCH 6/8] fix --- crates/rollup-boost/src/health.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index 730dc7ce..689c209c 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -9,7 +9,7 @@ use tokio::{ task::JoinHandle, time::{Instant, sleep_until}, }; -use tracing::{info, warn}; +use tracing::{warn}; use crate::{EngineApiExt, ExecutionMode, Health, Probes}; From d25d8b064a373d1e5130c0c9006b7f0626776de8 Mon Sep 17 00:00:00 2001 From: Karan Kurbur Date: Thu, 30 Oct 2025 15:34:11 -0700 Subject: [PATCH 7/8] add l2 client unsafe interval test --- crates/rollup-boost/src/health.rs | 59 ++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/crates/rollup-boost/src/health.rs b/crates/rollup-boost/src/health.rs index 689c209c..045f958b 100644 --- a/crates/rollup-boost/src/health.rs +++ b/crates/rollup-boost/src/health.rs @@ -9,7 +9,7 @@ use tokio::{ task::JoinHandle, time::{Instant, sleep_until}, }; -use tracing::{warn}; +use tracing::warn; use crate::{EngineApiExt, ExecutionMode, Health, Probes}; @@ -321,7 +321,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; @@ -363,7 +363,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; @@ -373,6 +373,48 @@ mod tests { Ok(()) } + #[serial] + #[tokio::test] + async fn test_health_check_l2_exceeds_max_unsafe_interval() -> eyre::Result<()> { + let probes = Arc::new(Probes::default()); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + + // L2 healthy unhealth + let l2 = MockHttpServer::serve(handler, now - 10).await.unwrap(); + let l2_client = Arc::new(RpcClient::new( + format!("http://{}", l2.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::L2, + )?); + + // Builder healthy + let builder = MockHttpServer::serve(handler, now).await.unwrap(); + let builder_client = Arc::new(RpcClient::new( + format!("http://{}", builder.addr).parse::()?, + JwtSecret::random(), + 100, + PayloadSource::Builder, + )?); + + let health_handle = HealthHandle { + probes: probes.clone(), + execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), + l2_client: l2_client.clone(), + builder_client: builder_client.clone(), + health_check_interval: Duration::from_secs(60), + max_unsafe_interval: 5, + }; + + health_handle.spawn(); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!(matches!(probes.health(), Health::ServiceUnavailable)); + Ok(()) + } + #[serial] #[tokio::test] async fn test_health_check_exceeds_max_unsafe_interval_execution_mode_disabled() @@ -404,7 +446,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::Disabled)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; @@ -445,7 +487,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::DryRun)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; @@ -485,7 +527,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; @@ -505,9 +547,8 @@ mod tests { .as_secs(); // L2 returns an error - let l2 = MockHttpServer::serve(error_handler, now).await.unwrap(); let l2_client = Arc::new(RpcClient::new( - format!("http://{}", l2.addr).parse::()?, + "http://127.0.0.1:6000".parse::()?, JwtSecret::random(), 100, PayloadSource::L2, @@ -527,7 +568,7 @@ mod tests { execution_mode: Arc::new(Mutex::new(ExecutionMode::Enabled)), l2_client: l2_client.clone(), builder_client: builder_client.clone(), - health_check_interval: Duration::from_secs(1), + health_check_interval: Duration::from_secs(60), max_unsafe_interval: 5, }; From de99a886c4f2000587da2970448000412d95de53 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 5 Nov 2025 10:16:24 -0800 Subject: [PATCH 8/8] chore: specify serial_test version --- crates/rollup-boost/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rollup-boost/Cargo.toml b/crates/rollup-boost/Cargo.toml index 58378628..49967399 100644 --- a/crates/rollup-boost/Cargo.toml +++ b/crates/rollup-boost/Cargo.toml @@ -66,7 +66,7 @@ parking_lot = "0.12.3" tokio-util = { version = "0.7.13" } [dev-dependencies] -serial_test = "*" +serial_test = "3" rand = "0.9.0" time = { version = "0.3.36", features = ["macros", "formatting", "parsing"] } op-alloy-consensus = "0.17.2"