Add benchmark machine placeholder (#11198)

* Move new_rng to shared code

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add bechmark machine command

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add --no-hardware-benchmarks

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Lockfile

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Do not create components if not needed

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Revert "Add --no-hardware-benchmarks"

This reverts commit d4ee982.

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update Cargo deps

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move sr255119::verify bench to sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move sr255119::verify bench to sc-sysinfo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Switch benchmarks to return f64

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Hide command until completed

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use concrete rand implementation

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Put clobber into a function

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add comment

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update cargo to match polkadot

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove doc that does not format in the console

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Limit benchmark by time

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add ExecutionLimit and make function infallible

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

bin/node-template/node/src/command.rs

@@ -109,8 +109,6 @@ pub fn run() -> sc_cli::Result<()> {
 			let runner = cli.create_runner(cmd)?;
 
 			runner.sync_run(|config| {
-				let PartialComponents { client, backend, .. } = service::new_partial(&config)?;
-
 				// This switch needs to be in the client, since the client decides
 				// which sub-commands it wants to support.
 				match cmd {
@@ -125,18 +123,25 @@ pub fn run() -> sc_cli::Result<()> {
 
 						cmd.run::<Block, service::ExecutorDispatch>(config)
 					},
-					BenchmarkCmd::Block(cmd) => cmd.run(client),
+					BenchmarkCmd::Block(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
+						cmd.run(client)
+					},
 					BenchmarkCmd::Storage(cmd) => {
+						let PartialComponents { client, backend, .. } =
+							service::new_partial(&config)?;
 						let db = backend.expose_db();
 						let storage = backend.expose_storage();
 
 						cmd.run(config, client, db, storage)
 					},
 					BenchmarkCmd::Overhead(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
 						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
 
 						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
 					},
+					BenchmarkCmd::Machine(cmd) => cmd.run(&config),
 				}
 			})
 		},

bin/node/cli/src/command.rs

@@ -99,8 +99,6 @@ pub fn run() -> Result<()> {
 			let runner = cli.create_runner(cmd)?;
 
 			runner.sync_run(|config| {
-				let PartialComponents { client, backend, .. } = new_partial(&config)?;
-
 				// This switch needs to be in the client, since the client decides
 				// which sub-commands it wants to support.
 				match cmd {
@@ -115,18 +113,24 @@ pub fn run() -> Result<()> {
 
 						cmd.run::<Block, ExecutorDispatch>(config)
 					},
-					BenchmarkCmd::Block(cmd) => cmd.run(client),
+					BenchmarkCmd::Block(cmd) => {
+						let PartialComponents { client, .. } = new_partial(&config)?;
+						cmd.run(client)
+					},
 					BenchmarkCmd::Storage(cmd) => {
+						let PartialComponents { client, backend, .. } = new_partial(&config)?;
 						let db = backend.expose_db();
 						let storage = backend.expose_storage();
 
 						cmd.run(config, client, db, storage)
 					},
 					BenchmarkCmd::Overhead(cmd) => {
+						let PartialComponents { client, .. } = new_partial(&config)?;
 						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
 
 						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
 					},
+					BenchmarkCmd::Machine(cmd) => cmd.run(&config),
 				}
 			})
 		},

bin/node/cli/tests/benchmark_block_works.rs

@@ -30,7 +30,7 @@ pub mod common;
 async fn benchmark_block_works() {
 	let base_dir = tempdir().expect("could not create a temp dir");
 
-	common::run_node_for_a_while(base_dir.path(), &["--dev"]).await;
+	common::run_node_for_a_while(base_dir.path(), &["--dev", "--no-hardware-benchmarks"]).await;
 
 	// Invoke `benchmark block` with all options to make sure that they are valid.
 	let status = Command::new(cargo_bin("substrate"))

bin/node/cli/tests/benchmark_machine_works.rs

@@ -0,0 +1,32 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::process::Command;
+
+/// Tests that the `benchmark machine` command works for the substrate dev runtime.
+#[test]
+fn benchmark_machine_works() {
+	let status = Command::new(cargo_bin("substrate"))
+		.args(["benchmark", "machine", "--dev"])
+		.args(["--verify-duration", "0.1"])
+		.status()
+		.unwrap();
+
+	assert!(status.success());
+}

client/sysinfo/Cargo.toml

@@ -23,4 +23,6 @@ libc = "0.2"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.79"
 sp-core = { version = "6.0.0", path = "../../primitives/core" }
+sp-io = { version = "6.0.0", path = "../../primitives/io" }
+sp-std = { version = "4.0.0", path = "../../primitives/std" }
 sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" }

client/sysinfo/src/lib.rs

@@ -20,12 +20,16 @@
 //! and software telemetry information about the node on which we're running.
 
 use futures::prelude::*;
+use std::time::Duration;
 
 mod sysinfo;
 #[cfg(target_os = "linux")]
 mod sysinfo_linux;
 
-pub use sysinfo::{gather_hwbench, gather_sysinfo};
+pub use sysinfo::{
+	benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
+	benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo,
+};
 
 /// The operating system part of the current target triplet.
 pub const TARGET_OS: &str = include_str!(concat!(env!("OUT_DIR"), "/target_os.txt"));
@@ -49,6 +53,38 @@ pub struct HwBench {
 	pub disk_random_write_score: Option<u64>,
 }
 
+/// Limit the execution time of a benchmark.
+pub enum ExecutionLimit {
+	/// Limit by the maximal duration.
+	MaxDuration(Duration),
+
+	/// Limit by the maximal number of iterations.
+	MaxIterations(usize),
+
+	/// Limit by the maximal duration and maximal number of iterations.
+	Both { max_iterations: usize, max_duration: Duration },
+}
+
+impl ExecutionLimit {
+	/// Returns the duration limit or `MAX` if none is present.
+	pub fn max_duration(&self) -> Duration {
+		match self {
+			Self::MaxDuration(d) => *d,
+			Self::Both { max_duration, .. } => *max_duration,
+			_ => Duration::from_secs(u64::MAX),
+		}
+	}
+
+	/// Returns the iterations limit or `MAX` if none is present.
+	pub fn max_iterations(&self) -> usize {
+		match self {
+			Self::MaxIterations(d) => *d,
+			Self::Both { max_iterations, .. } => *max_iterations,
+			_ => usize::MAX,
+		}
+	}
+}
+
 /// Prints out the system software/hardware information in the logs.
 pub fn print_sysinfo(sysinfo: &sc_telemetry::SysInfo) {
 	log::info!("💻 Operating system: {}", TARGET_OS);