subsystem-bench: add regression tests for availability read and write

.gitlab/pipeline/test.yml

@@ -25,6 +25,7 @@ test-linux-stable:
     # "upgrade_version_checks_should_work" is currently failing
     - |
       time cargo nextest run \
+        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \
@@ -68,7 +69,7 @@ test-linux-stable-runtime-benchmarks:
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
   script:
-    - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
+    - time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
 
 # can be used to run all tests
 # test-linux-stable-all:
@@ -492,3 +493,16 @@ test-syscalls:
       printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
       fi
   allow_failure: false # this rarely triggers in practice
+
+subsystem-regression-tests:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  script:
+    # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks
+    - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks
+    - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks
+  tags:
+    - benchmark

polkadot/node/core/approval-voting/Cargo.toml

@@ -53,3 +53,14 @@ kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
 log = { workspace = true, default-features = true }
 env_logger = "0.9.0"
+
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[test]]
+name = "approval-voting-regression-bench"
+path = "tests/approval-voting-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs

@@ -0,0 +1,176 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use polkadot_subsystem_bench::{
+	approval::{bench_approvals, prepare_test, ApprovalsOptions},
+	configuration::TestConfiguration,
+	usage::BenchmarkUsage,
+};
+
+const BENCH_COUNT: usize = 3;
+const WARM_UP_COUNT: usize = 20;
+const WARM_UP_PRECISION: f64 = 0.01;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+
+	messages.extend(approvals_no_shows()?);
+	// messages.extend(approvals_throughput()?);
+	// messages.extend(approvals_throughput_best_case()?);
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}
+
+fn test_configuration() -> TestConfiguration {
+	let mut config = TestConfiguration::default();
+	config.n_validators = 500;
+	config.n_cores = 100;
+	config.min_pov_size = 1120;
+	config.max_pov_size = 5120;
+	config.peer_bandwidth = 524288000000;
+	config.bandwidth = 524288000000;
+	config.num_blocks = 10;
+	config.generate_pov_sizes();
+
+	config
+}
+
+fn warm_up(test_case: &str, options: ApprovalsOptions) -> Result<(), String> {
+	println!("Warming up...");
+	let mut prev_run: Option<BenchmarkUsage> = None;
+	for _ in 0..WARM_UP_COUNT {
+		let curr = run(test_case, options.clone());
+		if let Some(ref prev) = prev_run {
+			let ap_distr_diff =
+				curr.cpu_usage_diff(prev, "approval-distribution").expect("Must exist");
+			let ap_vot_diff = curr.cpu_usage_diff(prev, "approval-voting").expect("Must exist");
+			if ap_distr_diff < WARM_UP_PRECISION && ap_vot_diff < WARM_UP_PRECISION {
+				return Ok(())
+			}
+		}
+		prev_run = Some(curr);
+	}
+
+	Err("Can't warm up".to_string())
+}
+
+fn benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage {
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> =
+		(0..BENCH_COUNT).map(|_| run(test_case, options.clone())).collect();
+	let usage = BenchmarkUsage::average(&usages);
+	println!("{}", usage);
+	usage
+}
+
+fn run(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage {
+	let (mut env, state) = prepare_test(test_configuration(), options.clone(), false);
+	env.runtime().block_on(bench_approvals(test_case, &mut env, state))
+}
+
+fn approvals_no_shows() -> Result<Vec<String>, String> {
+	let mut messages = vec![];
+	let test_case = "approvals_no_shows";
+	let options = ApprovalsOptions {
+		coalesce_mean: 3.0,
+		coalesce_std_dev: 1.0,
+		enable_assignments_v2: true,
+		last_considered_tranche: 89,
+		stop_when_approved: true,
+		coalesce_tranche_diff: 12,
+		num_no_shows_per_candidate: 10,
+		workdir_prefix: "/tmp".to_string(),
+	};
+
+	warm_up(test_case, options.clone())?;
+	let usage = benchmark(test_case, options.clone());
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 7000.0, 0.05),
+		("Sent to peers", 8100.0, 0.05),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 0.990, 0.05),
+		("approval-voting", 1.310, 0.05),
+	]));
+
+	Ok(messages)
+}
+
+#[allow(dead_code)]
+fn approvals_throughput() -> Result<Vec<String>, String> {
+	let mut messages = vec![];
+	let test_case = "approvals_throughput";
+	let options = ApprovalsOptions {
+		coalesce_mean: 3.0,
+		coalesce_std_dev: 1.0,
+		enable_assignments_v2: true,
+		last_considered_tranche: 89,
+		stop_when_approved: false,
+		coalesce_tranche_diff: 12,
+		num_no_shows_per_candidate: 0,
+		workdir_prefix: "/tmp".to_string(),
+	};
+
+	warm_up(test_case, options.clone())?;
+	let usage = benchmark(test_case, options.clone());
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 52850.0, 0.05),
+		("Sent to peers", 63500.0, 0.05),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 8.645, 0.05),
+		("approval-voting", 11.085, 0.05),
+	]));
+
+	Ok(messages)
+}
+
+#[allow(dead_code)]
+fn approvals_throughput_best_case() -> Result<Vec<String>, String> {
+	let mut messages = vec![];
+	let test_case = "approvals_throughput_best_case";
+	let options = ApprovalsOptions {
+		coalesce_mean: 3.0,
+		coalesce_std_dev: 1.0,
+		enable_assignments_v2: true,
+		last_considered_tranche: 89,
+		stop_when_approved: true,
+		coalesce_tranche_diff: 12,
+		num_no_shows_per_candidate: 0,
+		workdir_prefix: "/tmp".to_string(),
+	};
+
+	warm_up(test_case, options.clone())?;
+	let usage = benchmark(test_case, options.clone());
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 2950.0, 0.05),
+		("Sent to peers", 3250.0, 0.05),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 0.48, 0.05),
+		("approval-voting", 0.625, 0.05),
+	]));
+
+	Ok(messages)
+}

polkadot/node/network/availability-distribution/Cargo.toml

@@ -36,3 +36,14 @@ sc-network = { path = "../../../../substrate/client/network" }
 futures-timer = "3.0.2"
 assert_matches = "1.4.0"
 polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+
+[[test]]
+name = "availability-distribution-regression-bench"
+path = "tests/availability-distribution-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs

@@ -0,0 +1,102 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use polkadot_subsystem_bench::{
+	availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState},
+	configuration::{PeerLatency, TestConfiguration},
+	usage::BenchmarkUsage,
+};
+
+const BENCH_COUNT: usize = 3;
+const WARM_UP_COUNT: usize = 20;
+const WARM_UP_PRECISION: f64 = 0.01;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.latency = Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 });
+	config.n_validators = 1000;
+	config.n_cores = 200;
+	config.max_validators_per_core = 5;
+	config.min_pov_size = 5120;
+	config.max_pov_size = 5120;
+	config.peer_bandwidth = 52428800;
+	config.bandwidth = 52428800;
+	config.connectivity = 75;
+	config.num_blocks = 3;
+	config.generate_pov_sizes();
+
+	warm_up(config.clone())?;
+	let usage = benchmark(config.clone());
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 4330.0, 0.05),
+		("Sent to peers", 15900.0, 0.05),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("availability-distribution", 0.025, 0.05),
+		("bitfield-distribution", 0.085, 0.05),
+		("availability-store", 0.180, 0.05),
+	]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}
+
+fn warm_up(config: TestConfiguration) -> Result<(), String> {
+	println!("Warming up...");
+	let mut prev_run: Option<BenchmarkUsage> = None;
+	for _ in 0..WARM_UP_COUNT {
+		let curr = run(config.clone());
+		if let Some(ref prev) = prev_run {
+			let av_distr_diff =
+				curr.cpu_usage_diff(prev, "availability-distribution").expect("Must exist");
+			let bitf_distr_diff =
+				curr.cpu_usage_diff(prev, "bitfield-distribution").expect("Must exist");
+			let av_store_diff =
+				curr.cpu_usage_diff(prev, "availability-store").expect("Must exist");
+			if av_distr_diff < WARM_UP_PRECISION &&
+				bitf_distr_diff < WARM_UP_PRECISION &&
+				av_store_diff < WARM_UP_PRECISION
+			{
+				return Ok(())
+			}
+		}
+		prev_run = Some(curr);
+	}
+
+	Err("Can't warm up".to_string())
+}
+
+fn benchmark(config: TestConfiguration) -> BenchmarkUsage {
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT).map(|_| run(config.clone())).collect();
+	let usage = BenchmarkUsage::average(&usages);
+	println!("{}", usage);
+	usage
+}
+
+fn run(config: TestConfiguration) -> BenchmarkUsage {
+	let mut state = TestState::new(&config);
+	let (mut env, _protocol_config) =
+		prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false);
+	env.runtime()
+		.block_on(benchmark_availability_write("data_availability_write", &mut env, state))
+}

polkadot/node/network/availability-recovery/Cargo.toml

@@ -41,6 +41,13 @@ sc-network = { path = "../../../../substrate/client/network" }
 
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[test]]
+name = "availability-recovery-regression-bench"
+path = "tests/availability-recovery-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
 
 [features]
 subsystem-benchmarks = []