Adjust quarantine tests (#3233)

* Reuse the same poldercast ids when we restart a node

* Adjust quarantine tests timings

If a node is restarted almost immediately then it's higly
probable that it won't be quarantined by other peers.
Let's wait a little bit more so that we test quarantine
behavior as well.

* Reuse the same poldercast ids for integration tests

* Store topology key in NodeSettings

testing/jormungandr-integration-tests/src/common/network/builder.rs

@@ -2,6 +2,7 @@ use super::{Controller, ControllerError};
 use chain_addr::Discrimination;
 use chain_impl_mockchain::value::Value;
 use chain_impl_mockchain::{chaintypes::ConsensusVersion, milli::Milli};
+use jormungandr_lib::crypto::key::SigningKey;
 use jormungandr_lib::interfaces::{
     ActiveSlotCoefficient, KesUpdateSpeed, NodeSecret, NumberOfSlotsPerEpoch, SlotDuration,
 };
@@ -72,6 +73,7 @@ impl NetworkBuilder {
                             bft: None,
                             genesis: None,
                         },
+                        topology_secret: SigningKey::generate(&mut rand::thread_rng()),
                         node_topology: template,
                     },
                 )
@@ -151,7 +153,3 @@ pub fn wallet(alias: &str) -> WalletTemplateBuilder {
         discrimination: Discrimination::Test,
     }
 }
-
-pub fn params(alias: &str) -> SpawnParams {
-    SpawnParams::new(&alias)
-}

testing/jormungandr-integration-tests/src/common/network/controller.rs

@@ -12,12 +12,16 @@ use jormungandr_testing_utils::{
     wallet::Wallet,
 };
 
-use assert_fs::fixture::FixtureError;
+use assert_fs::fixture::{ChildPath, FixtureError};
 use assert_fs::prelude::*;
 use assert_fs::TempDir;
 use std::path::PathBuf;
 use thiserror::Error;
 
+const NODE_CONFIG_FILE: &str = "node_config.yaml";
+const NODE_SECRETS_FILE: &str = "node_secret.yaml";
+const NODE_TOPOLOGY_KEY_FILE: &str = "node_topology_key";
+
 #[derive(Error, Debug)]
 pub enum ControllerError {
     #[error("node not found {0}")]
@@ -80,6 +84,17 @@ impl Controller {
         }
     }
 
+    pub fn spawn_params(&self, alias: &str) -> SpawnParams {
+        let mut spawn_params = SpawnParams::new(alias);
+        spawn_params.node_key_file(
+            self.node_dir(alias)
+                .child(NODE_TOPOLOGY_KEY_FILE)
+                .path()
+                .into(),
+        );
+        spawn_params
+    }
+
     pub fn spawn_and_wait(&mut self, alias: &str) -> JormungandrProcess {
         self.spawn_node(alias, PersistenceMode::InMemory, LeadershipMode::Leader)
             .unwrap_or_else(|_| panic!("cannot start {}", alias))
@@ -119,9 +134,13 @@ impl Controller {
         Ok(process)
     }
 
+    fn node_dir(&self, alias: &str) -> ChildPath {
+        self.working_directory.child(alias)
+    }
+
     fn make_starter_for(&mut self, spawn_params: &SpawnParams) -> Result<Starter, ControllerError> {
         let node_setting = self.node_settings(&spawn_params.alias)?;
-        let dir = self.working_directory.child(&node_setting.alias);
+        let dir = self.node_dir(&node_setting.alias);
         let mut config = node_setting.config().clone();
         spawn_params.override_settings(&mut config);
 
@@ -142,14 +161,17 @@ impl Controller {
         }
         dir.create_dir_all()?;
 
-        let config_file = dir.child("node_config.yaml");
+        let config_file = dir.child(NODE_CONFIG_FILE);
         let yaml = serde_yaml::to_string(&config)?;
         config_file.write_str(&yaml)?;
 
-        let secret_file = dir.child("node_secret.yaml");
+        let secret_file = dir.child(NODE_SECRETS_FILE);
         let yaml = serde_yaml::to_string(node_setting.secrets())?;
         secret_file.write_str(&yaml)?;
 
+        let topology_file = dir.child(NODE_TOPOLOGY_KEY_FILE);
+        topology_file.write_str(&node_setting.topology_secret.to_bech32_str())?;
+
         let params = JormungandrParams::new(
             config,
             config_file.path(),
@@ -177,7 +199,7 @@ impl Controller {
         leadership_mode: LeadershipMode,
     ) -> Result<JormungandrProcess, ControllerError> {
         self.spawn_custom(
-            SpawnParams::new(alias)
+            self.spawn_params(alias)
                 .leadership_mode(leadership_mode)
                 .persistence_mode(persistence_mode),
         )

testing/jormungandr-integration-tests/src/common/network/mod.rs

@@ -1,5 +1,5 @@
 mod builder;
 mod controller;
 
-pub use builder::{builder, params, wallet};
+pub use builder::{builder, wallet};
 pub use controller::{Controller, ControllerError};

testing/jormungandr-integration-tests/src/networking/p2p.rs

@@ -1,6 +1,6 @@
 use crate::common::{
     jormungandr::process::JormungandrProcess,
-    network::{self, params, wallet},
+    network::{self, wallet},
 };
 
 use jormungandr_lib::{
@@ -9,6 +9,7 @@ use jormungandr_lib::{
     },
     time::Duration,
 };
+use jormungandr_testing_utils::testing::network_builder::SpawnParams;
 use jortestkit::process as process_utils;
 const CLIENT: &str = "CLIENT";
 const SERVER: &str = "SERVER";
@@ -129,7 +130,9 @@ pub fn node_whitelist_itself() {
             wallet("delegated1").with(1_000_000).delegated_to(CLIENT),
             wallet("delegated2").with(1_000_000).delegated_to(SERVER),
         ])
-        .custom_config(vec![params(CLIENT).explorer(Explorer { enabled: true })])
+        .custom_config(vec![
+            SpawnParams::new(CLIENT).explorer(Explorer { enabled: true })
+        ])
         .build()
         .unwrap();
 
@@ -146,7 +149,7 @@ pub fn node_whitelist_itself() {
     };
 
     let client = network_controller
-        .spawn_custom(params(CLIENT).policy(policy))
+        .spawn_custom(network_controller.spawn_params(CLIENT).policy(policy))
         .unwrap();
     client.assert_no_errors_in_log();
 }
@@ -159,7 +162,9 @@ pub fn node_does_not_quarantine_whitelisted_node() {
             wallet("delegated1").with(1_000_000).delegated_to(CLIENT),
             wallet("delegated2").with(1_000_000).delegated_to(SERVER),
         ])
-        .custom_config(vec![params(CLIENT).explorer(Explorer { enabled: true })])
+        .custom_config(vec![
+            SpawnParams::new(CLIENT).explorer(Explorer { enabled: true })
+        ])
         .build()
         .unwrap();
 
@@ -176,7 +181,7 @@ pub fn node_does_not_quarantine_whitelisted_node() {
     };
 
     let client = network_controller
-        .spawn_custom(params(CLIENT).policy(policy))
+        .spawn_custom(network_controller.spawn_params(CLIENT).policy(policy))
         .unwrap();
 
     server.shutdown();
@@ -200,7 +205,9 @@ pub fn node_put_in_quarantine_nodes_which_are_not_whitelisted() {
             wallet("delegated1").with(1_000_000).delegated_to(CLIENT),
             wallet("delegated2").with(1_000_000).delegated_to(SERVER),
         ])
-        .custom_config(vec![params(CLIENT).explorer(Explorer { enabled: true })])
+        .custom_config(vec![
+            SpawnParams::new(CLIENT).explorer(Explorer { enabled: true })
+        ])
         .build()
         .unwrap();
 
@@ -217,7 +224,7 @@ pub fn node_put_in_quarantine_nodes_which_are_not_whitelisted() {
     };
 
     let client = network_controller
-        .spawn_custom(params(CLIENT).policy(policy))
+        .spawn_custom(network_controller.spawn_params(CLIENT).policy(policy))
         .unwrap();
 
     server.shutdown();
@@ -257,21 +264,25 @@ pub fn node_trust_itself() {
             wallet("delegated1").with(1_000_000).delegated_to(CLIENT),
             wallet("delegated2").with(1_000_000).delegated_to(SERVER),
         ])
-        .custom_config(vec![params(CLIENT).explorer(Explorer { enabled: true })])
+        .custom_config(vec![
+            SpawnParams::new(CLIENT).explorer(Explorer { enabled: true })
+        ])
         .build()
         .unwrap();
 
     let _server = network_controller.spawn_and_wait(SERVER);
 
-    let config = network_controller.node_config(CLIENT).unwrap().p2p.clone();
+    let config = network_controller.node_config(CLIENT).unwrap().p2p;
 
     let peer = TrustedPeer {
         address: config.public_address,
         id: None,
     };
     network_controller
         .expect_spawn_failed(
-            params(CLIENT).trusted_peers(vec![peer]),
+            network_controller
+                .spawn_params(CLIENT)
+                .trusted_peers(vec![peer]),
             "failed to retrieve the list of bootstrap peers from trusted peer",
         )
         .unwrap();
@@ -291,7 +302,7 @@ pub fn node_put_itself_in_preffered_layers() {
 
     let _server = network_controller.spawn_and_wait(SERVER);
 
-    let config = network_controller.node_config(CLIENT).unwrap().p2p.clone();
+    let config = network_controller.node_config(CLIENT).unwrap().p2p;
 
     let peer = TrustedPeer {
         address: config.public_address,
@@ -305,7 +316,9 @@ pub fn node_put_itself_in_preffered_layers() {
 
     assert!(network_controller
         .expect_spawn_failed(
-            params(CLIENT).preferred_layer(layer),
+            network_controller
+                .spawn_params(CLIENT)
+                .preferred_layer(layer),
             "topology tells the node to connect to itself"
         )
         .is_ok());
@@ -339,8 +352,8 @@ pub fn topic_of_interest_influences_node_sync_ability() {
                 .delegated_to(slow_client_alias),
         ])
         .custom_config(vec![
-            params(fast_client_alias).topics_of_interest(high_topic_of_interests),
-            params(slow_client_alias).topics_of_interest(low_topic_of_interests),
+            SpawnParams::new(fast_client_alias).topics_of_interest(high_topic_of_interests),
+            SpawnParams::new(slow_client_alias).topics_of_interest(low_topic_of_interests),
         ])
         .build()
         .unwrap();

testing/jormungandr-scenario-tests/src/node.rs

@@ -180,6 +180,7 @@ pub struct Node {
 
 const NODE_CONFIG: &str = "node_config.yaml";
 const NODE_SECRET: &str = "node_secret.yaml";
+const NODE_TOPOLOGY_KEY: &str = "node_topology_key";
 const NODE_STORAGE: &str = "storage.db";
 
 impl NodeController {
@@ -758,6 +759,13 @@ impl<'a, R: RngCore, N> SpawnBuilder<'a, R, N> {
         })
     }
 
+    fn write_topology_file<P: AsRef<Path>>(&self, key_file: P) -> Result<()> {
+        Ok(std::fs::write(
+            key_file.as_ref(),
+            self.node_settings.topology_secret.to_bech32_str(),
+        )?)
+    }
+
     fn apply_persistence_setting(&mut self, dir: &Path) {
         if self.peristence_mode == PersistenceMode::Persistent {
             let path_to_storage = dir.join(NODE_STORAGE);
@@ -795,13 +803,17 @@ impl<'a, R: RngCore, N> SpawnBuilder<'a, R, N> {
 
 impl<'a, R: RngCore> SpawnBuilder<'a, R, Node> {
     pub fn build(mut self) -> Result<Node> {
-        let dir = self.working_dir.join(self.alias.to_owned());
+        let dir = self.working_dir.clone();
         std::fs::DirBuilder::new().recursive(true).create(&dir)?;
 
         let config_file = dir.join(NODE_CONFIG);
         let config_secret = dir.join(NODE_SECRET);
+        let topology_key_file = dir.join(NODE_TOPOLOGY_KEY);
 
         self.apply_persistence_setting(&dir);
+
+        self.node_settings.config.p2p.node_key_file = Some(topology_key_file.clone());
+        self.write_topology_file(&topology_key_file)?;
         self.write_config_file(&config_file)?;
         self.write_secret_file(&config_secret)?;
 

testing/jormungandr-scenario-tests/src/scenario/controller.rs

@@ -122,8 +122,10 @@ impl ControllerBuilder {
             _ => (),
         }
 
+        let settings = self.settings.unwrap();
+
         Controller::new(
-            self.settings.unwrap(),
+            settings,
             context,
             working_directory,
             self.blockchain.unwrap(),
@@ -329,7 +331,13 @@ impl Controller {
     }
 
     pub fn new_spawn_params(&self, node_alias: &str) -> SpawnParams {
-        SpawnParams::new(node_alias)
+        let mut spawn_params = SpawnParams::new(node_alias);
+        spawn_params.node_key_file(self.node_dir(node_alias).path().into());
+        spawn_params
+    }
+
+    fn node_dir(&self, alias: &str) -> ChildPath {
+        self.working_directory.child(alias)
     }
 
     pub fn spawn_legacy_node(
@@ -370,7 +378,7 @@ impl Controller {
             .progress_bar(pb)
             .alias(params.get_alias())
             .block0(block0_setting)
-            .working_dir(self.working_directory.path())
+            .working_dir(self.node_dir(&params.get_alias()).path())
             .peristence_mode(params.get_persistence_mode());
         let node = spawn_builder.build(version)?;
         Ok(node.controller())
@@ -414,7 +422,7 @@ impl Controller {
             .progress_bar(pb)
             .alias(params.get_alias())
             .block0(block0_setting)
-            .working_dir(self.working_directory.path())
+            .working_dir(self.node_dir(&params.get_alias()).path())
             .peristence_mode(params.get_persistence_mode());
         let node = spawn_builder.build()?;
 

testing/jormungandr-scenario-tests/src/scenario/settings.rs

@@ -8,7 +8,7 @@ use chain_impl_mockchain::{
     vote::PayloadType,
 };
 
-use jormungandr_lib::crypto::account::Identifier;
+use jormungandr_lib::crypto::{account::Identifier, key::SigningKey};
 use jormungandr_lib::interfaces::try_initials_vec_from_messages;
 use jormungandr_lib::{
     interfaces::{
@@ -272,6 +272,7 @@ impl PrepareNodeSettings for NodeSetting {
             alias,
             config: NodeConfig::prepare(context),
             secret: NodeSecret::prepare(context),
+            topology_secret: SigningKey::generate(&mut rand::thread_rng()),
             node_topology: template.clone(),
         }
     }

testing/jormungandr-scenario-tests/src/test/non_functional/disruption.rs

@@ -137,6 +137,9 @@ pub fn passive_leader_disruption_overlap(
     // 2. Only leader is up
     passive.shutdown()?;
 
+    // Wait a bit so that the leader can indeed notice that passive is offline
+    utils::wait(15);
+
     // 3. Both nodes are up
     let passive = controller.spawn_node(
         PASSIVE,
@@ -206,7 +209,7 @@ pub fn leader_leader_disruption_overlap(mut context: Context<ChaChaRng>) -> Resu
 
     // 3. second node is down
     leader2.shutdown()?;
-    utils::wait(5);
+    utils::wait(15);
 
     // 4. both nodes are up
     let leader2 = controller.spawn_node(

testing/jormungandr-testing-utils/src/testing/network_builder/settings.rs

@@ -34,6 +34,8 @@ pub struct NodeSetting {
 
     pub config: NodeConfig,
 
+    pub topology_secret: SigningKey<Ed25519>,
+
     pub node_topology: NodeTemplate,
 }
 
@@ -42,12 +44,14 @@ impl NodeSetting {
         alias: NodeAlias,
         config: NodeConfig,
         secret: NodeSecret,
+        topology_secret: SigningKey<Ed25519>,
         template: NodeTemplate,
     ) -> Self {
         Self {
             alias,
             config,
             secret,
+            topology_secret,
             node_topology: template,
         }
     }