Add explorer to hersir (#4108)

* Refactor hersir: pull out all keys generation and allow to define external wallets or committess and voteplans, this will allow external clients to create their own keys which will be used for tally operations.

* update tests with hersir changes

* Refactor explorer module. Extract configuration struct for explorer. Added extension to multiaddr struct for better address formatting and finally make ExplorerProcess constructor return error

* added simple explorer client which is capable to send simple sanity request to explorer server

* update usages for ExplorerProcess new constructor impl

* added possibility to start explorer by hersir

src/jormungandr/explorer/src/tests.rs

@@ -33,7 +33,9 @@ pub fn explorer_test_context(
 
     let params = ExplorerParams::new(test_config.query_complexity_limit, None, None);
 
-    (jormungandr.explorer(params), jormungandr)
+    let explorer_process = jormungandr.explorer(params).unwrap();
+    println!("{:?}", explorer_process.client().current_time());
+    (explorer_process, jormungandr)
 }
 
 pub fn get_invalid_block(explorer: &Explorer) {

src/jormungandr/testing/hersir/res/example.yaml

@@ -10,6 +10,11 @@ nodes:
         leadership_mode: leader
         persistence_mode: inmemory
 
+explorer:
+    connect_to: passive
+    persistence_mode: inmemory
+
+
 blockchain:
     discrimination: test
     consensus: bft

src/jormungandr/testing/hersir/src/builder/explorer.rs

@@ -0,0 +1,30 @@
+use crate::{builder::NodeSetting, config::ExplorerTemplate};
+use jormungandr_automation::{
+    jormungandr::{explorer::configuration::ExplorerConfiguration, get_available_port, NodeAlias},
+    utils::MultiaddrExtension,
+};
+use std::{collections::HashMap, path::Path};
+
+pub fn generate_explorer(
+    nodes: &HashMap<NodeAlias, NodeSetting>,
+    explorer_template: &ExplorerTemplate,
+) -> Result<ExplorerConfiguration, Error> {
+    let settings = nodes
+        .get(&explorer_template.connect_to)
+        .ok_or_else(|| Error::CannotFindAlias(explorer_template.connect_to.clone()))?;
+
+    Ok(ExplorerConfiguration {
+        explorer_port: get_available_port(),
+        explorer_listen_address: "127.0.0.1".to_string(),
+        node_address: settings.config.p2p.public_address.clone().to_http_addr(),
+        logs_dir: Some(Path::new("C:\\work\\iohk\\logs.txt").to_path_buf()),
+        storage_dir: None,
+        params: explorer_template.to_explorer_params(),
+    })
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("cannot find alias '{0}' for any defined node")]
+    CannotFindAlias(NodeAlias),
+}

src/jormungandr/testing/hersir/src/builder/mod.rs

@@ -1,4 +1,5 @@
 mod committee;
+mod explorer;
 pub mod rng;
 pub mod settings;
 mod stake_pool;
@@ -9,7 +10,8 @@ pub mod wallet;
 pub use crate::controller::Error as ControllerError;
 use crate::{
     config::{
-        Blockchain, CommitteeTemplate, Config, SessionSettings, VotePlanTemplate, WalletTemplate,
+        Blockchain, CommitteeTemplate, Config, ExplorerTemplate, SessionSettings, VotePlanTemplate,
+        WalletTemplate,
     },
     controller::{Controller, Error},
     utils::Dotifier,
@@ -35,6 +37,7 @@ pub struct NetworkBuilder {
     topology: Topology,
     blockchain: Blockchain,
     session_settings: SessionSettings,
+    explorer_template: Option<ExplorerTemplate>,
     wallet_templates: Vec<WalletTemplate>,
     committee_templates: Vec<CommitteeTemplate>,
     vote_plan_templates: Vec<VotePlanTemplate>,
@@ -72,6 +75,7 @@ impl NetworkBuilder {
             .wallet_templates(config.wallets)
             .vote_plan_templates(config.vote_plans)
             .committees(config.committees)
+            .explorer(config.explorer)
     }
 
     pub fn topology(mut self, topology: Topology) -> Self {
@@ -142,6 +146,7 @@ impl NetworkBuilder {
             &self.blockchain,
             &self.wallet_templates,
             &self.committee_templates,
+            &self.explorer_template,
             &self.vote_plan_templates,
             &mut random,
         )?;
@@ -155,6 +160,11 @@ impl NetworkBuilder {
         self.finish_all();
         Controller::new(settings, self.session_settings.root)
     }
+
+    pub fn explorer(mut self, explorer: Option<ExplorerTemplate>) -> Self {
+        self.explorer_template = explorer;
+        self
+    }
 }
 
 fn document(path: &Path, settings: &Settings) -> Result<(), Error> {

src/jormungandr/testing/hersir/src/builder/settings/mod.rs

@@ -5,15 +5,18 @@ pub(crate) mod wallet;
 pub use crate::{builder::settings::node::NodeSetting, config::Blockchain};
 use crate::{
     builder::{
-        committee::generate_committee_data, stake_pool, vote::generate_vote_plans,
-        wallet::generate, Node as NodeTemplate, Random, VotePlanKey, VotePlanSettings, Wallet,
+        committee::generate_committee_data, explorer::generate_explorer, stake_pool,
+        vote::generate_vote_plans, wallet::generate, Node as NodeTemplate, Random, VotePlanKey,
+        VotePlanSettings, Wallet,
     },
-    config::{CommitteeTemplate, VotePlanTemplate, WalletTemplate},
+    config::{CommitteeTemplate, ExplorerTemplate, VotePlanTemplate, WalletTemplate},
 };
 use assert_fs::fixture::{ChildPath, PathChild};
 use chain_crypto::Ed25519;
 use chain_impl_mockchain::{chaintypes::ConsensusVersion, fee::LinearFee};
-use jormungandr_automation::jormungandr::NodeAlias;
+use jormungandr_automation::jormungandr::{
+    explorer::configuration::ExplorerConfiguration, NodeAlias,
+};
 use jormungandr_lib::{
     crypto::key::SigningKey,
     interfaces::{
@@ -31,6 +34,7 @@ pub struct Settings {
     pub wallets: Vec<Wallet>,
     pub committees: Vec<CommitteeIdDef>,
     pub block0: Block0Configuration,
+    pub explorer: Option<ExplorerConfiguration>,
     pub stake_pools: HashMap<NodeAlias, StakePool>,
     pub vote_plans: HashMap<VotePlanKey, VotePlanSettings>,
 }
@@ -41,6 +45,7 @@ impl Settings {
         blockchain: &Blockchain,
         wallets: &[WalletTemplate],
         committees: &[CommitteeTemplate],
+        explorer: &Option<ExplorerTemplate>,
         vote_plans: &[VotePlanTemplate],
         rng: &mut Random<RNG>,
     ) -> Result<Self, Error>
@@ -59,6 +64,7 @@ impl Settings {
                 ),
                 initial: Vec::new(),
             },
+            explorer: None,
             stake_pools: HashMap::new(),
             vote_plans: HashMap::new(),
         };
@@ -71,6 +77,11 @@ impl Settings {
 
         settings.vote_plans = vote_plans;
         let discrimination = settings.block0.blockchain_configuration.discrimination;
+
+        if let Some(explorer) = explorer {
+            settings.explorer = Some(generate_explorer(&settings.nodes, explorer)?);
+        }
+
         settings.block0.initial.extend(
             fragments
                 .iter()
@@ -198,4 +209,6 @@ pub enum Error {
     Settings(#[from] wallet::Error),
     #[error(transparent)]
     Committee(#[from] crate::builder::committee::Error),
+    #[error(transparent)]
+    Explorer(#[from] crate::builder::explorer::Error),
 }

src/jormungandr/testing/hersir/src/config/mod.rs

@@ -14,7 +14,9 @@ use crate::{
     builder::{Node, NodeAlias, Topology},
     error::Error,
 };
-use jormungandr_automation::jormungandr::{LogLevel, TestingDirectory};
+use jormungandr_automation::jormungandr::{
+    explorer::configuration::ExplorerParams, LogLevel, PersistenceMode, TestingDirectory,
+};
 use serde::Deserialize;
 use std::{collections::HashSet, path::PathBuf, str::FromStr};
 pub use vote_plan::VotePlanTemplate;
@@ -23,6 +25,7 @@ pub use vote_plan::VotePlanTemplate;
 pub struct Config {
     pub blockchain: Blockchain,
     pub nodes: Vec<NodeConfig>,
+    pub explorer: Option<ExplorerTemplate>,
     #[serde(default)]
     pub session: SessionSettings,
     pub wallets: Vec<WalletTemplate>,
@@ -108,6 +111,30 @@ fn default_title() -> String {
     "unnamed_scenario".to_owned()
 }
 
+#[derive(Debug, Deserialize, Clone)]
+pub struct ExplorerTemplate {
+    pub connect_to: NodeAlias,
+    #[serde(default = "default_persistence_mode")]
+    pub persistence_mode: PersistenceMode,
+    pub address_bech32_prefix: Option<String>,
+    pub query_depth_limit: Option<u64>,
+    pub query_complexity_limit: Option<u64>,
+}
+
+fn default_persistence_mode() -> PersistenceMode {
+    PersistenceMode::InMemory
+}
+
+impl ExplorerTemplate {
+    pub fn to_explorer_params(&self) -> ExplorerParams {
+        ExplorerParams {
+            address_bech32_prefix: self.address_bech32_prefix.clone(),
+            query_complexity_limit: self.query_complexity_limit,
+            query_depth_limit: self.query_depth_limit,
+        }
+    }
+}
+
 impl Default for SessionSettings {
     fn default() -> Self {
         Self {

src/jormungandr/testing/hersir/src/controller/error.rs

@@ -64,4 +64,6 @@ pub enum Error {
     SettingsWallet(#[from] crate::builder::settings::wallet::Error),
     #[error(transparent)]
     Settings(#[from] crate::builder::settings::Error),
+    #[error("no explorer configuration defined")]
+    NoExplorerConfigurationDefined,
 }

src/jormungandr/testing/hersir/src/controller/interactive/args/explorer.rs

@@ -34,7 +34,7 @@ impl ExplorerTip {
             })?;
         println!(
             "{:#?}",
-            node.explorer(ExplorerParams::default())
+            node.explorer(ExplorerParams::default())?
                 .client()
                 .last_block()?
         );

src/jormungandr/testing/hersir/src/controller/mod.rs

@@ -19,7 +19,7 @@ pub use interactive::{
     UserInteractionController,
 };
 use jormungandr_automation::jormungandr::{
-    ConfiguredStarter, JormungandrParams, JormungandrProcess, LegacyNodeConfig,
+    ConfiguredStarter, ExplorerProcess, JormungandrParams, JormungandrProcess, LegacyNodeConfig,
     LegacyNodeConfigConverter, LogLevel, NodeAlias, PersistenceMode, Starter, TestingDirectory,
     Version,
 };
@@ -197,6 +197,16 @@ impl Controller {
         builder.build()
     }
 
+    pub fn spawn_explorer(&mut self) -> Result<ExplorerProcess, Error> {
+        ExplorerProcess::new(
+            self.settings
+                .explorer
+                .clone()
+                .ok_or(Error::NoExplorerConfigurationDefined)?,
+        )
+        .map_err(Into::into)
+    }
+
     pub fn spawn_node_async(&mut self, alias: &str) -> Result<JormungandrProcess, Error> {
         let mut starter = self.make_starter_for(
             SpawnParams::new(alias).persistence_mode(PersistenceMode::InMemory),

src/jormungandr/testing/hersir/src/controller/monitor/node/mod.rs

@@ -170,7 +170,7 @@ impl Node {
         multiaddr::to_tcp_socket_addr(&self.process.p2p_public_address()).unwrap()
     }
 
-    pub fn explorer(&self) -> ExplorerProcess {
+    pub fn explorer(&self) -> Result<ExplorerProcess, ExplorerError> {
         self.process.explorer(ExplorerParams::default())
     }
 
@@ -310,6 +310,7 @@ impl Node {
     }
 }
 
+use jormungandr_automation::jormungandr::ExplorerError;
 use std::fmt::Display;
 
 impl ProgressBarController {

src/jormungandr/testing/hersir/src/spawn/standard.rs

@@ -35,6 +35,16 @@ pub fn spawn_network(config: Config, args: Args) -> Result<(), Error> {
         println!("Node '{}' started", alias);
     }
 
+    let _maybe_explorer = {
+        if controller.settings().explorer.is_some() {
+            let explorer = Some(controller.spawn_explorer()?);
+            println!("explorer started");
+            explorer
+        } else {
+            None
+        }
+    };
+
     println!("Network is started");
     loop {
         for node in processes.values() {

src/jormungandr/testing/jormungandr-automation/Cargo.toml

@@ -15,6 +15,7 @@ futures = "0.3.21"
 base64 = "0.13"
 bech32 = "0.8"
 bytesize = "1.1.0"
+structopt = "0.3.26"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chain-impl-mockchain = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master", features = [ "property-test-api" ] }
@@ -82,3 +83,4 @@ property-test-api = [ ]
 
 [build-dependencies]
 tonic-build = "0.6"
+versionisator = "1.0.2"

src/jormungandr/testing/jormungandr-automation/build.rs

@@ -9,4 +9,22 @@ fn main() {
     println!("cargo:rustc-env=JORMUNGANDR_NAME={}", jormungandr_name);
     println!("cargo:rustc-env=JOR_EXPLORER_NAME={}", jor_explorer_name);
     println!("cargo:rustc-env=RUST_BACKTRACE=full");
+
+    let pkg_version = if let Ok(date) = std::env::var("DATE") {
+        format!("{}.{}", env!("CARGO_PKG_VERSION"), date)
+    } else {
+        env!("CARGO_PKG_VERSION").to_string()
+    };
+
+    println!("cargo:rustc-env=CARGO_PKG_VERSION={}", pkg_version);
+
+    let version = versionisator::Version::new(
+        env!("CARGO_MANIFEST_DIR"),
+        env!("CARGO_PKG_NAME").to_string(),
+        pkg_version,
+    );
+
+    println!("cargo:rustc-env=FULL_VERSION={}", version.full());
+    println!("cargo:rustc-env=SIMPLE_VERSION={}", version.simple());
+    println!("cargo:rustc-env=SOURCE_VERSION={}", version.hash());
 }