From 58bd898f5be362d7c67f8ba36eeac172482f4072 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 28 Apr 2023 11:13:21 -0300 Subject: [PATCH] feat(zebrad): Refuse to run zebrad when release is too old (#6351) * refuse to run Zebra if it is too old * update the release checklist to consider the constants * bring newline back * apply new end of support code * attempt to add tests (not working yet) * move eos to progress task * move tests * add acceptance test (not working) * fix tests * change to block height checks (ugly code) * change warn days * refactor estimated blocks per day, etc * move end of support code to its own task * change test * fix some docs * move constants * remove uneeded conversions * downgrade tracing * reduce end of support time, fix ci changing debugs to info again * update instructions * add failure messages * cargo lock update * unify releaase name constant * change info msg * clippy fixes * add a block explorer * ignore testnet in end of support task * change panic to 16 weeks * add some documentation about end of support * Tweak docs wording --------- Co-authored-by: teor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../release-checklist.md | 17 ++- Cargo.lock | 24 +++++ book/src/dev/release-process.md | 7 +- book/src/user/run.md | 4 + zebra-network/src/constants.rs | 15 ++- zebra-rpc/src/methods.rs | 2 +- zebra-rpc/src/methods/tests/snapshot.rs | 2 +- zebra-rpc/src/methods/tests/vectors.rs | 2 +- zebrad/Cargo.toml | 2 + zebrad/src/application.rs | 10 +- zebrad/src/commands/start.rs | 13 ++- zebrad/src/components/sync.rs | 1 + zebrad/src/components/sync/end_of_support.rs | 102 ++++++++++++++++++ zebrad/tests/acceptance.rs | 26 +++++ zebrad/tests/common/failure_messages.rs | 3 + .../common/lightwalletd/wallet_grpc_test.rs | 2 +- zebrad/tests/end_of_support.rs | 92 ++++++++++++++++ 17 files changed, 308 insertions(+), 16 deletions(-) create mode 100644 zebrad/src/components/sync/end_of_support.rs create mode 100644 zebrad/tests/end_of_support.rs diff --git a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md index d1d1b5bb35d..a6552418e02 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release-checklist.md +++ b/.github/PULL_REQUEST_TEMPLATE/release-checklist.md @@ -39,7 +39,7 @@ Once you know which versions you want to increment, you can find them in the: zebrad (rc): - [ ] zebrad `Cargo.toml` -- [ ] `zebra-network` protocol user agent: https://github.com/ZcashFoundation/zebra/blob/main/zebra-network/src/constants.rs +- [ ] `zebra-network` release version (`RELEASE_VERSION`): https://github.com/ZcashFoundation/zebra/blob/main/zebra-network/src/constants.rs - [ ] `README.md` - [ ] `book/src/user/docker.md` @@ -123,6 +123,19 @@ From "Keep a Changelog": +## Release support constants + +Needed for the end of support feature. Please update the following constants [in this file](https://github.com/ZcashFoundation/zebra/blob/main/zebrad/src/components/sync/end_of_support.rs): + +- [ ] `ESTIMATED_RELEASE_HEIGHT` (required) - Replace with the estimated height you estimate the release will be tagged. +
+ - Find where the Zcash blockchain tip is now by using a [Zcash explorer](https://zcashblockexplorer.com/blocks) or other tool. + - Consider there are aprox `1152` blocks per day (with the current Zcash `75` seconds spacing). + - So for example if you think the release will be tagged somewhere in the next 3 days you can add `1152 * 3` to the current tip height and use that value here. +
+ +- [ ] `EOS_PANIC_AFTER` (optional) - Replace if you want the release to be valid for a different numbers of days into the future. The default here is 120 days. + ## Create the Release ### Create the Release PR @@ -131,7 +144,7 @@ After you have the version increments, the updated checkpoints, any missed depen and the updated changelog: - [ ] Make sure the PRs with the new checkpoint hashes and missed dependencies are already merged -- [ ] Push the version increments and the updated changelog into a branch +- [ ] Push the version increments, the updated changelog and the release constants into a branch (for example: `bump-v1.0.0-rc.0` - this needs to be different to the tag name) - [ ] Create a release PR by adding `&template=release-checklist.md` to the comparing url ([Example](https://github.com/ZcashFoundation/zebra/compare/v1.0.0-rc.0-release?expand=1&template=release-checklist.md)). - [ ] Add the list of deleted changelog entries as a comment to make reviewing easier. diff --git a/Cargo.lock b/Cargo.lock index 3c1cfb06f41..f4c7729d56f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5211,6 +5211,29 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-test" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber 0.3.17", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" +dependencies = [ + "lazy_static", + "quote 1.0.26", + "syn 1.0.109", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -6246,6 +6269,7 @@ dependencies = [ "tracing-futures", "tracing-journald", "tracing-subscriber 0.3.17", + "tracing-test", "vergen", "zebra-chain", "zebra-consensus", diff --git a/book/src/dev/release-process.md b/book/src/dev/release-process.md index fe026435f7e..f8eba0f7184 100644 --- a/book/src/dev/release-process.md +++ b/book/src/dev/release-process.md @@ -33,10 +33,11 @@ The pre-release version is denoted by appending a hyphen and a series of dot sep ### Supported Releases -Older Zebra versions are always supported until the next Zebra major, minor or patch release. Initially, we can only guarantee support for the latest Zebra release. -We might be able to provide support for earlier releases, or we might ask you to upgrade to the latest release. +Every Zebra version released by the Zcash Foundation is supported up to a specific height. Currently we support each version for about **16 weeks** but this can change from release to release. -Our support periods will be extended as we gain experience with supporting Zebra releases. +When the Zcash chain reaches this end of support height, `zebrad` will shut down and the binary will refuse to start. + +Our process is similar to `zcashd`: https://zcash.github.io/zcash/user/release-support.html Older Zebra versions that only support previous network upgrades will never be supported, because they are operating on an unsupported Zcash chain fork. diff --git a/book/src/user/run.md b/book/src/user/run.md index c161e2cc396..c4b85ab2283 100644 --- a/book/src/user/run.md +++ b/book/src/user/run.md @@ -11,6 +11,10 @@ structure, and documentation for all of the config options can be found * `zebrad start` starts a full node. +## Supported versions + +Always run a supported version of Zebra, and upgrade it regularly, so it doesn't become unsupported and halt. [More information](https://github.com/ZcashFoundation/zebra/blob/main/book/src/dev/release-process.md#supported-releases). + ## Return Codes - `0`: Application exited successfully diff --git a/zebra-network/src/constants.rs b/zebra-network/src/constants.rs index 927f7cc2cdf..ba6da14b34a 100644 --- a/zebra-network/src/constants.rs +++ b/zebra-network/src/constants.rs @@ -269,14 +269,12 @@ pub const MAX_ADDRS_IN_ADDRESS_BOOK: usize = /// messages from each of our peers. pub const TIMESTAMP_TRUNCATION_SECONDS: u32 = 30 * 60; -/// The User-Agent string provided by the node. +/// Release version name is used to form user agent string. +/// Can be also used in other parts of Zebra to identify the current release. /// -/// This must be a valid [BIP 14] user agent. -/// -/// [BIP 14]: https://github.com/bitcoin/bips/blob/master/bip-0014.mediawiki // // TODO: generate this from crate metadata (#2375) -pub const USER_AGENT: &str = "/Zebra:1.0.0-rc.7/"; +pub const RELEASE_VERSION: &str = "1.0.0-rc.7"; /// The Zcash network protocol version implemented by this crate, and advertised /// during connection setup. @@ -336,6 +334,13 @@ lazy_static! { } else { Regex::new("(access a socket in a way forbidden by its access permissions)|(Only one usage of each socket address)") }.expect("regex is valid"); + + /// The User-Agent string provided by the node. + /// + /// This must be a valid [BIP 14] user agent. + /// + /// [BIP 14]: https://github.com/bitcoin/bips/blob/master/bip-0014.mediawiki + pub static ref USER_AGENT: String = format!("/Zebra:{RELEASE_VERSION}/"); } /// The timeout for DNS lookups. diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index f76847a1cd1..b23bbc9d379 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -368,7 +368,7 @@ where fn get_info(&self) -> Result { let response = GetInfo { build: self.app_version.clone(), - subversion: USER_AGENT.into(), + subversion: USER_AGENT.to_string(), }; Ok(response) diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index 21a8c584a04..e28af914ff2 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -267,7 +267,7 @@ fn snapshot_rpc_getinfo(info: GetInfo, settings: &insta::Settings) { insta::assert_json_snapshot!("get_info", info, { ".subversion" => dynamic_redaction(|value, _path| { // assert that the subversion value is user agent - assert_eq!(value.as_str().unwrap(), USER_AGENT); + assert_eq!(value.as_str().unwrap(), USER_AGENT.to_string()); // replace with: "[SubVersion]" }), diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index c2f2c0c258d..5ec79b7dbc0 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -46,7 +46,7 @@ async fn rpc_getinfo() { // make sure there is a `subversion` field, // and that is equal to the Zebra user agent. - assert_eq!(get_info.subversion, USER_AGENT); + assert_eq!(get_info.subversion, USER_AGENT.to_string()); mempool.expect_no_requests().await; state.expect_no_requests().await; diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index 3bf57149cbb..340a95c7f2c 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -203,6 +203,8 @@ serde_json = { version = "1.0.96", features = ["preserve_order"] } tempfile = "3.5.0" hyper = { version = "0.14.26", features = ["http1", "http2", "server"]} +tracing-test = { version = "0.2.4", features = ["no-env-filter"] } + tokio = { version = "1.27.0", features = ["full", "tracing", "test-util"] } tokio-stream = "0.1.14" diff --git a/zebrad/src/application.rs b/zebrad/src/application.rs index 4408de1119a..fab48a59000 100644 --- a/zebrad/src/application.rs +++ b/zebrad/src/application.rs @@ -13,7 +13,11 @@ use abscissa_core::{ use zebra_network::constants::PORT_IN_USE_ERROR; use zebra_state::constants::{DATABASE_FORMAT_VERSION, LOCK_FILE_ERROR}; -use crate::{commands::ZebradCmd, components::tracing::Tracing, config::ZebradConfig}; +use crate::{ + commands::ZebradCmd, + components::{sync::end_of_support::EOS_PANIC_MESSAGE_HEADER, tracing::Tracing}, + config::ZebradConfig, +}; mod entry_point; use entry_point::EntryPoint; @@ -294,6 +298,10 @@ impl Application for ZebradApp { if LOCK_FILE_ERROR.is_match(error_str) { return false; } + // Don't ask users to report old version panics. + if error_str.to_string().contains(EOS_PANIC_MESSAGE_HEADER) { + return false; + } true } color_eyre::ErrorKind::Recoverable(error) => { diff --git a/zebrad/src/commands/start.rs b/zebrad/src/commands/start.rs index 7a1f95999fc..ddff2f21dee 100644 --- a/zebrad/src/commands/start.rs +++ b/zebrad/src/commands/start.rs @@ -232,12 +232,16 @@ impl StartCmd { let progress_task_handle = tokio::spawn( show_block_chain_progress( config.network.network, - latest_chain_tip, + latest_chain_tip.clone(), sync_status.clone(), ) .in_current_span(), ); + let end_of_support_task_handle = tokio::spawn( + sync::end_of_support::start(config.network.network, latest_chain_tip).in_current_span(), + ); + // Give the inbound service more time to clear its queue, // then start concurrent tasks that can add load to the inbound service // (by opening more peer connections, so those peers send us requests) @@ -267,6 +271,7 @@ impl StartCmd { pin!(mempool_queue_checker_task_handle); pin!(tx_gossip_task_handle); pin!(progress_task_handle); + pin!(end_of_support_task_handle); // startup tasks let BackgroundTaskHandles { @@ -334,6 +339,11 @@ impl StartCmd { .expect("unexpected panic in the chain progress task"); } + end_of_support_result = &mut end_of_support_task_handle => end_of_support_result + .expect("unexpected panic in the end of support task") + .map(|_| info!("end of support task exited")), + + // Unlike other tasks, we expect the download task to finish while Zebra is running. groth16_download_result = &mut groth16_download_handle_fused => { groth16_download_result @@ -389,6 +399,7 @@ impl StartCmd { mempool_queue_checker_task_handle.abort(); tx_gossip_task_handle.abort(); progress_task_handle.abort(); + end_of_support_task_handle.abort(); // startup tasks groth16_download_handle.abort(); diff --git a/zebrad/src/components/sync.rs b/zebrad/src/components/sync.rs index c0cc3889e56..8679f66d6b4 100644 --- a/zebrad/src/components/sync.rs +++ b/zebrad/src/components/sync.rs @@ -27,6 +27,7 @@ use crate::{ }; mod downloads; +pub mod end_of_support; mod gossip; mod progress; mod recent_sync_lengths; diff --git a/zebrad/src/components/sync/end_of_support.rs b/zebrad/src/components/sync/end_of_support.rs new file mode 100644 index 00000000000..bad99cf133b --- /dev/null +++ b/zebrad/src/components/sync/end_of_support.rs @@ -0,0 +1,102 @@ +//! End of support checking task. + +use std::time::Duration; + +use color_eyre::Report; +use lazy_static::lazy_static; + +use zebra_chain::{ + block::Height, + chain_tip::ChainTip, + parameters::{Network, NetworkUpgrade}, +}; + +use zebra_network::constants::RELEASE_VERSION; + +lazy_static! { + /// The name of the current Zebra release. + pub static ref RELEASE_NAME: String = format!("Zebra {}", RELEASE_VERSION); +} + +/// The estimated height that this release started to run. +pub const ESTIMATED_RELEASE_HEIGHT: u32 = 2_026_000; + +/// The maximum number of days after `ESTIMATED_RELEASE_HEIGHT` where a Zebra server will run +/// without halting. +/// +/// Notes: +/// +/// - Zebra will exit with a panic if the current tip height is bigger than the `ESTIMATED_RELEASE_HEIGHT` +/// plus this number of days. +pub const EOS_PANIC_AFTER: u32 = 112; + +/// The number of days before the end of support where Zebra will display warnings. +pub const EOS_WARN_AFTER: u32 = EOS_PANIC_AFTER - 14; + +/// A string which is part of the panic that will be displayed if Zebra halts. +pub const EOS_PANIC_MESSAGE_HEADER: &str = "Zebra refuses to run"; + +/// A string which is part of the warning that will be displayed if Zebra release is close to halting. +pub const EOS_WARN_MESSAGE_HEADER: &str = "Your Zebra release is too old and it will stop running"; + +/// The amount of time between end of support checks. +const CHECK_INTERVAL: Duration = Duration::from_secs(60 * 60); + +/// Wait a few seconds at startup so `best_tip_height` is always `Some`. +const INITIAL_WAIT: Duration = Duration::from_secs(10); + +/// Start the end of support checking task for Mainnet. +pub async fn start( + network: Network, + latest_chain_tip: impl ChainTip + std::fmt::Debug, +) -> Result<(), Report> { + info!("Starting end of support task"); + + tokio::time::sleep(INITIAL_WAIT).await; + + loop { + if network == Network::Mainnet { + if let Some(tip_height) = latest_chain_tip.best_tip_height() { + check(tip_height, network); + } + } else { + info!("Release always valid in Testnet"); + } + tokio::time::sleep(CHECK_INTERVAL).await; + } +} + +/// Check if the current release is too old and panic if so. +pub fn check(tip_height: Height, network: Network) { + info!("Checking if Zebra release is inside support range ..."); + + // Get the current block spacing + let target_block_spacing = NetworkUpgrade::target_spacing_for_height(network, tip_height); + + // Get the number of blocks per day + let estimated_blocks_per_day = + u32::try_from(chrono::Duration::days(1).num_seconds() / target_block_spacing.num_seconds()) + .expect("number is always small enough to fit"); + + let panic_height = + Height(ESTIMATED_RELEASE_HEIGHT + (EOS_PANIC_AFTER * estimated_blocks_per_day)); + let warn_height = + Height(ESTIMATED_RELEASE_HEIGHT + (EOS_WARN_AFTER * estimated_blocks_per_day)); + + if tip_height > panic_height { + panic!( + "{EOS_PANIC_MESSAGE_HEADER} if the release date is older than {EOS_PANIC_AFTER} days. \ + \nRelease name: {}, Estimated release height: {ESTIMATED_RELEASE_HEIGHT} \ + \nHint: Download and install the latest Zebra release from: https://github.com/ZcashFoundation/zebra/releases/latest", + *RELEASE_NAME + ); + } else if tip_height > warn_height { + warn!( + "{EOS_WARN_MESSAGE_HEADER} at block {}. \ + \nRelease name: {}, Estimated release height: {ESTIMATED_RELEASE_HEIGHT} \ + \nHint: Download and install the latest Zebra release from: https://github.com/ZcashFoundation/zebra/releases/latest", panic_height.0, RELEASE_NAME.to_string() + ); + } else { + info!("Zebra release is supported until block {}, please report bugs at https://github.com/ZcashFoundation/zebra/issues", panic_height.0); + } +} diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index 5db592146ef..81952672cf7 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -2214,6 +2214,32 @@ async fn submit_block() -> Result<()> { common::get_block_template_rpcs::submit_block::run().await } +/// Check that the the end of support code is called at least once. +#[test] +fn end_of_support_is_checked_at_start() -> Result<()> { + let _init_guard = zebra_test::init(); + let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let mut child = testdir.spawn_child(args!["start"])?; + + // Give enough time to start up the eos task. + std::thread::sleep(Duration::from_secs(30)); + + child.kill(false)?; + + let output = child.wait_with_output()?; + let output = output.assert_failure()?; + + // Zebra started + output.stdout_line_contains("Starting zebrad")?; + + // End of support task started. + output.stdout_line_contains("Starting end of support task")?; + + // Make sure the command was killed + output.assert_was_killed()?; + + Ok(()) +} /// Test `zebra-checkpoints` on mainnet. /// /// If you want to run this test individually, see the module documentation. diff --git a/zebrad/tests/common/failure_messages.rs b/zebrad/tests/common/failure_messages.rs index 1b6ff6f986b..f6543ec939b 100644 --- a/zebrad/tests/common/failure_messages.rs +++ b/zebrad/tests/common/failure_messages.rs @@ -40,6 +40,9 @@ pub const ZEBRA_FAILURE_MESSAGES: &[&str] = &[ // TODO: log these errors in Zebra, and check for them in the Zebra logs? "Invalid params", "Method not found", + // Logs related to end of support halting feature. + zebrad::components::sync::end_of_support::EOS_PANIC_MESSAGE_HEADER, + zebrad::components::sync::end_of_support::EOS_WARN_MESSAGE_HEADER, ]; /// Failure log messages from lightwalletd. diff --git a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs index c583863215c..6771958ca6f 100644 --- a/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs +++ b/zebrad/tests/common/lightwalletd/wallet_grpc_test.rs @@ -358,7 +358,7 @@ pub async fn run() -> Result<()> { let lightd_info = rpc_client.get_lightd_info(Empty {}).await?.into_inner(); // Make sure the subversion field is zebra the user agent - assert_eq!(lightd_info.zcashd_subversion, USER_AGENT); + assert_eq!(lightd_info.zcashd_subversion, USER_AGENT.to_string()); Ok(()) } diff --git a/zebrad/tests/end_of_support.rs b/zebrad/tests/end_of_support.rs new file mode 100644 index 00000000000..42040c33815 --- /dev/null +++ b/zebrad/tests/end_of_support.rs @@ -0,0 +1,92 @@ +//! Testing the end of support feature. + +use std::time::Duration; + +use color_eyre::eyre::Result; +use tokio::time::timeout; + +use zebra_chain::{block::Height, chain_tip::mock::MockChainTip, parameters::Network}; + +use zebra_consensus::CheckpointList; + +use zebrad::components::sync::end_of_support::{self, EOS_PANIC_AFTER, ESTIMATED_RELEASE_HEIGHT}; + +// Estimated blocks per day with the current 75 seconds block spacing. +const ESTIMATED_BLOCKS_PER_DAY: u32 = 1152; + +/// Test that the `end_of_support` function is working as expected. +#[test] +#[should_panic(expected = "Zebra refuses to run if the release date is older than")] +fn end_of_support_panic() { + // We are in panic + let panic = ESTIMATED_RELEASE_HEIGHT + (EOS_PANIC_AFTER * ESTIMATED_BLOCKS_PER_DAY) + 1; + + end_of_support::check(Height(panic), Network::Mainnet); +} + +/// Test that the `end_of_support` function is working as expected. +#[test] +#[tracing_test::traced_test] +fn end_of_support_function() { + // We are away from warn or panic + let no_warn = ESTIMATED_RELEASE_HEIGHT + (EOS_PANIC_AFTER * ESTIMATED_BLOCKS_PER_DAY) + - (30 * ESTIMATED_BLOCKS_PER_DAY); + + end_of_support::check(Height(no_warn), Network::Mainnet); + assert!(logs_contain( + "Checking if Zebra release is inside support range ..." + )); + assert!(logs_contain("Zebra release is supported")); + + // We are in warn range + let warn = ESTIMATED_RELEASE_HEIGHT + (EOS_PANIC_AFTER * 1152) - (3 * ESTIMATED_BLOCKS_PER_DAY); + + end_of_support::check(Height(warn), Network::Mainnet); + assert!(logs_contain( + "Checking if Zebra release is inside support range ..." + )); + assert!(logs_contain( + "Your Zebra release is too old and it will stop running at block" + )); + + // Panic is tested in `end_of_support_panic` +} + +/// Test that we are never in end of support warning or panic. +#[test] +#[tracing_test::traced_test] +fn end_of_support_date() { + // Get the list of checkpoints. + let list = CheckpointList::new(Network::Mainnet); + + // Get the last one we have and use it as tip. + let higher_checkpoint = list.max_height(); + + end_of_support::check(higher_checkpoint, Network::Mainnet); + assert!(logs_contain( + "Checking if Zebra release is inside support range ..." + )); + assert!(!logs_contain( + "Your Zebra release is too old and it will stop running in" + )); +} + +/// Check that the the end of support task is working. +#[tokio::test] +#[tracing_test::traced_test] +async fn end_of_support_task() -> Result<()> { + let (latest_chain_tip, latest_chain_tip_sender) = MockChainTip::new(); + latest_chain_tip_sender.send_best_tip_height(Height(10)); + + let eos_future = end_of_support::start(Network::Mainnet, latest_chain_tip); + + let _ = timeout(Duration::from_secs(15), eos_future).await.ok(); + + assert!(logs_contain( + "Checking if Zebra release is inside support range ..." + )); + + assert!(logs_contain("Zebra release is supported")); + + Ok(()) +}