Added retry for jormungandr startup if port is not available

Signed-off-by: dkijania <dariusz.kijania@iohk.io>

Cargo.toml

@@ -77,12 +77,10 @@ quickcheck = "0.8"
 galvanic-test = "0.2.0"
 assert_cmd = "0.11"
 assert_fs = "0.11"
-fs_extra = "1.1.0"
 mktemp = "0.4.0"
 lazy_static = "1.3"
+custom_error = "1.6"
 
-[build-dependencies]
-fs_extra = "1.1.0"
 
 [features]
 with-bench = []

tests/common/configuration/jormungandr_config.rs

@@ -2,13 +2,17 @@
 
 use common::configuration::genesis_model::GenesisYaml;
 use common::configuration::node_config_model::NodeConfig;
+use common::configuration::secret_model::SecretModel;
 use std::path::PathBuf;
 
 #[derive(Debug)]
 pub struct JormungandrConfig {
     pub genesis_block_path: PathBuf,
+    pub node_config_path: PathBuf,
+    pub secret_model_path: PathBuf,
     pub genesis_yaml: GenesisYaml,
     pub node_config: NodeConfig,
+    pub secret_model: SecretModel,
 }
 
 impl JormungandrConfig {
@@ -17,18 +21,17 @@ impl JormungandrConfig {
     }
 
     pub fn new() -> JormungandrConfig {
-        JormungandrConfig {
-            genesis_block_path: PathBuf::from(""),
-            genesis_yaml: GenesisYaml::new(),
-            node_config: NodeConfig::new(),
-        }
+        JormungandrConfig::from(GenesisYaml::new(), NodeConfig::new())
     }
 
     pub fn from(genesis_yaml: GenesisYaml, node_config: NodeConfig) -> JormungandrConfig {
         JormungandrConfig {
             genesis_block_path: PathBuf::from(""),
+            node_config_path: PathBuf::from(""),
+            secret_model_path: PathBuf::from(""),
             genesis_yaml: genesis_yaml,
             node_config: node_config,
+            secret_model: SecretModel::empty(),
         }
     }
 }

tests/common/configuration/node_config_model.rs

@@ -48,8 +48,8 @@ impl NodeConfig {
     }
 
     pub fn new() -> NodeConfig {
-        let rest_port = get_available_port();
-        let public_address_port = get_available_port();
+        let rest_port = NodeConfig::get_available_port();
+        let public_address_port = NodeConfig::get_available_port();
         let storage_file = file_utils::get_path_in_temp("storage");
 
         NodeConfig {
@@ -72,25 +72,33 @@ impl NodeConfig {
         }
     }
 
+    pub fn regenerate_ports(&mut self) {
+        self.rest.listen = format!("127.0.0.1:{}", NodeConfig::get_available_port().to_string());
+        self.peer_2_peer.public_address = format!(
+            "/ip4/127.0.0.1/tcp/{}",
+            NodeConfig::get_available_port().to_string()
+        );
+    }
+
     pub fn get_node_address(&self) -> String {
         let output = format!("http://{}/{}", self.rest.listen, self.rest.prefix);
         output
     }
-}
 
-fn get_available_port() -> u16 {
-    let available_port = loop {
-        let port = rand::thread_rng().gen_range(8000, 9999);
-        if port_is_available(port) {
-            break port;
-        }
-    };
-    available_port
-}
+    fn get_available_port() -> u16 {
+        let available_port = loop {
+            let port = rand::thread_rng().gen_range(8000, 9999);
+            if NodeConfig::port_is_available(port) {
+                break port;
+            }
+        };
+        available_port
+    }
 
-fn port_is_available(port: u16) -> bool {
-    match TcpListener::bind(("127.0.0.1", port)) {
-        Ok(_) => true,
-        Err(_) => false,
+    fn port_is_available(port: u16) -> bool {
+        match TcpListener::bind(("127.0.0.1", port)) {
+            Ok(_) => true,
+            Err(_) => false,
+        }
     }
 }

tests/common/configuration/secret_model.rs

@@ -23,6 +23,10 @@ impl SecretModel {
         node_config_file_path
     }
 
+    pub fn empty() -> SecretModel {
+        SecretModel::new("")
+    }
+
     pub fn new(signing_key: &str) -> SecretModel {
         SecretModel {
             bft: BFT {

tests/common/process_utils/mod.rs

@@ -1,13 +1,20 @@
 #![allow(dead_code)]
 
+extern crate custom_error;
 extern crate serde_yaml;
 
 pub mod output_extensions;
 pub mod process_guard;
+
+use self::custom_error::custom_error;
 use self::output_extensions::ProcessOutput;
 use std::process::{Command, Output, Stdio};
 use std::{thread, time};
 
+custom_error! {pub ProcessError
+     ProcessExited{message: String} = "could not start process '{message}'",
+}
+
 /// Runs command, wait for output and returns it output
 ///
 /// # Arguments
@@ -108,7 +115,7 @@ pub fn run_process_until_response_matches<F: Fn(Output) -> bool>(
     max_attempts: i32,
     command_description: &str,
     error_description: &str,
-) {
+) -> Result<(), ProcessError> {
     let one_second = time::Duration::from_millis(&timeout * 1000);
     let mut attempts = max_attempts.clone();
 
@@ -146,10 +153,13 @@ pub fn run_process_until_response_matches<F: Fn(Output) -> bool>(
     }
 
     if attempts <= 0 {
-        panic!(
-            "{} (tried to connect {} times with {} s interval)",
-            &error_description, &max_attempts, &timeout
-        );
+        return Err(ProcessError::ProcessExited {
+            message: format!(
+                "{} (tried to connect {} times with {} s interval)",
+                &error_description, &max_attempts, &timeout
+            ),
+        });
     }
     println!("Success: {}", &command_description);
+    Ok(())
 }

tests/common/startup/jormungandr_starter.rs

@@ -0,0 +1,102 @@
+#![cfg(feature = "integration-test")]
+
+extern crate custom_error;
+
+use self::custom_error::custom_error;
+
+use common::configuration::jormungandr_config::JormungandrConfig;
+use common::jcli_wrapper;
+use common::jormungandr_wrapper;
+use common::process_utils;
+use common::process_utils::{
+    output_extensions::ProcessOutput, process_guard::ProcessKillGuard, ProcessError,
+};
+
+use std::process::{Command, Output};
+
+custom_error! {pub StartupError
+    JormungandrNotLaunched{ source: ProcessError } = "could not start jormungandr",
+}
+
+fn try_to_start_jormungandr_node(
+    rest_address: &str,
+    command: &mut Command,
+) -> Result<ProcessKillGuard, StartupError> {
+    println!("Starting jormungandr node...");
+    let process = command
+        .spawn()
+        .expect("failed to execute 'start jormungandr node'");
+
+    let guard = ProcessKillGuard::new(process, String::from("Jormungandr node"));
+
+    let proces_start_result = process_utils::run_process_until_response_matches(
+        jcli_wrapper::jcli_commands::get_rest_stats_command(&rest_address),
+        &is_node_up,
+        2,
+        5,
+        "get stats from jormungandr node",
+        "jormungandr node is not up",
+    );
+
+    match proces_start_result {
+        Ok(_) => return Ok(guard),
+        Err(e) => return Err(StartupError::JormungandrNotLaunched { source: e }),
+    }
+}
+
+fn start_jormungandr_node_sync_with_retry(
+    rest_address: &str,
+    command: &mut Command,
+    config: &mut JormungandrConfig,
+) -> ProcessKillGuard {
+    let first_attempt = try_to_start_jormungandr_node(rest_address, command);
+    match first_attempt {
+        Ok(guard) => return guard,
+        _ => (),
+    };
+    config.node_config.regenerate_ports();
+    let second_attempt = try_to_start_jormungandr_node(rest_address, command);
+    match second_attempt {
+        Ok(guard) => return guard,
+        Err(e) => panic!(e.to_string()),
+    };
+}
+
+fn is_node_up(output: Output) -> bool {
+    match output.as_single_node_yaml().get("uptime") {
+        Some(uptime) => {
+            return uptime
+                .parse::<i32>()
+                .expect(&format!("Cannot parse uptime {}", uptime.to_string()))
+                > 2
+        }
+        None => return false,
+    }
+}
+
+pub fn start_jormungandr_node(mut config: &mut JormungandrConfig) -> ProcessKillGuard {
+    let rest_address = &config.node_config.get_node_address();
+
+    let mut command = jormungandr_wrapper::get_start_jormungandr_node_command(
+        &config.node_config_path,
+        &config.genesis_block_path,
+    );
+
+    println!("Starting node with configuration : {:?}", &config);
+    let process = start_jormungandr_node_sync_with_retry(&rest_address, &mut command, &mut config);
+    process
+}
+
+pub fn start_jormungandr_node_as_leader(mut config: &mut JormungandrConfig) -> ProcessKillGuard {
+    let rest_address = &config.node_config.get_node_address();
+
+    let mut command = jormungandr_wrapper::get_start_jormungandr_as_leader_node_command(
+        &config.node_config_path,
+        &config.genesis_block_path,
+        &config.secret_model_path,
+    );
+
+    println!("Starting node with configuration : {:?}", &config);
+    let process = start_jormungandr_node_sync_with_retry(&rest_address, &mut command, &mut config);
+    process
+}