test: simulate for worker node heartbeat timeout

src/prost/Cargo.toml

@@ -13,7 +13,7 @@ pbjson = "0.5"
 prost = "0.11"
 prost-helpers = { path = "helpers" }
 serde = { version = "1", features = ["derive"] }
-tonic = { version = "0.2.14", package = "madsim-tonic" }
+tonic = { version = "0.2.18", package = "madsim-tonic" }
 
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }

src/tests/simulation/Cargo.toml

@@ -16,12 +16,12 @@ async-trait = "0.1"
 aws-sdk-s3 = { version = "0.2.17", package = "madsim-aws-sdk-s3" }
 clap = { version = "4", features = ["derive"] }
 console = "0.15"
-etcd-client = { version = "0.2.17", package = "madsim-etcd-client" }
+etcd-client = { version = "0.2.18", package = "madsim-etcd-client" }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 glob = "0.3"
 itertools = "0.10"
 lru = { git = "https://github.com/risingwavelabs/lru-rs.git", branch = "evict_by_timestamp" }
-madsim = "0.2.17"
+madsim = "0.2.18"
 paste = "1"
 pin-project = "1.0"
 pretty_assertions = "1"

src/tests/simulation/src/cluster.rs

@@ -411,6 +411,10 @@ impl Cluster {
                 }
                 nodes.push(format!("meta-{}", i));
             }
+            // don't kill all meta services
+            if nodes.len() == self.config.meta_nodes {
+                nodes.truncate(1);
+            }
         }
         if opts.kill_frontend {
             let rand = rand::thread_rng().gen_range(0..3);
@@ -454,7 +458,14 @@ impl Cluster {
             tracing::info!("kill {name}");
             madsim::runtime::Handle::current().kill(name);
 
-            let t = rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1));
+            let mut t =
+                rand::thread_rng().gen_range(Duration::from_secs(0)..Duration::from_secs(1));
+            // has a small chance to restart after a long time
+            // so that the node is expired and removed from the cluster
+            if rand::thread_rng().gen_bool(0.1) {
+                // max_heartbeat_interval_secs = 60
+                t += Duration::from_secs(20);
+            }
             tokio::time::sleep(t).await;
             tracing::info!("restart {name}");
             madsim::runtime::Handle::current().restart(name);

src/tests/simulation/src/risingwave.toml

@@ -2,6 +2,10 @@
 #
 # Note: this file is embedded in the binary and cannot be changed without recompiling.
 
+[meta]
+# a relatively small number to make it easier to timeout
+max_heartbeat_interval_secs = 10
+
 [system]
 barrier_interval_ms = 250
 checkpoint_frequency = 4

src/tests/simulation/src/slt.rs

@@ -158,7 +158,7 @@ pub async fn run_slt_task(cluster: Arc<Cluster>, glob: &str, opts: &KillOpts) {
                 continue;
             }
 
-            let should_kill = thread_rng().gen_ratio((opts.kill_rate * 1000.0) as u32, 1000);
+            let should_kill = thread_rng().gen_bool(opts.kill_rate as f64);
             // spawn a background task to kill nodes
             let handle = if should_kill {
                 let cluster = cluster.clone();