subsystem-bench: add regression tests for availability read and write (#3311)

### What's been done
- `subsystem-bench` has been split into two parts: a cli benchmark
runner and a library.
- The cli runner is quite simple. It just allows us to run `.yaml` based
test sequences. Now it should only be used to run benchmarks during
development.
- The library is used in the cli runner and in regression tests. Some
code is changed to make the library independent of the runner.
- Added first regression tests for availability read and write that
replicate existing test sequences.

### How we run regression tests
- Regression tests are simply rust integration tests without the
harnesses.
- They should only be compiled under the `subsystem-benchmarks` feature
to prevent them from running with other tests.
- This doesn't work when running tests with `nextest` in CI, so
additional filters have been added to the `nextest` runs.
- Each benchmark run takes a different time in the beginning, so we
"warm up" the tests until their CPU usage differs by only 1%.
- After the warm-up, we run the benchmarks a few more times and compare
the average with the exception using a precision.

### What is still wrong?
- I haven't managed to set up approval voting tests. The spread of their
results is too large and can't be narrowed down in a reasonable amount
of time in the warm-up phase.
- The tests start an unconfigurable prometheus endpoint inside, which
causes errors because they use the same 9999 port. I disable it with a
flag, but I think it's better to extract the endpoint launching outside
the test, as we already do with `valgrind` and `pyroscope`. But we still
use `prometheus` inside the tests.

### Future work
* https://github.com/paritytech/polkadot-sdk/issues/3528
* https://github.com/paritytech/polkadot-sdk/issues/3529
* https://github.com/paritytech/polkadot-sdk/issues/3530
* https://github.com/paritytech/polkadot-sdk/issues/3531

---------

Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>

polkadot/node/subsystem-bench/src/availability/cli.rs

@@ -1,37 +0,0 @@
-// Copyright (C) Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use serde::{Deserialize, Serialize};
-
-#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)]
-#[value(rename_all = "kebab-case")]
-#[non_exhaustive]
-pub enum NetworkEmulation {
-	Ideal,
-	Healthy,
-	Degraded,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)]
-#[clap(rename_all = "kebab-case")]
-#[allow(missing_docs)]
-pub struct DataAvailabilityReadOptions {
-	#[clap(short, long, default_value_t = false)]
-	/// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as
-	/// we don't need to re-construct from chunks. Tipically this is only faster if nodes have
-	/// enough bandwidth.
-	pub fetch_from_backers: bool,
-}

polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs

@@ -14,54 +14,32 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! A tool for running subsystem benchmark tests designed for development and
-//! CI regression testing.
+//! A tool for running subsystem benchmark tests
+//! designed for development and CI regression testing.
 
-use approval::{bench_approvals, ApprovalsOptions};
-use availability::{
-	cli::{DataAvailabilityReadOptions, NetworkEmulation},
-	prepare_test, TestState,
-};
 use clap::Parser;
-use clap_num::number_range;
 use color_eyre::eyre;
 use colored::Colorize;
-use core::{
-	configuration::TestConfiguration,
-	display::display_configuration,
-	environment::{TestEnvironment, GENESIS_HASH},
-};
+use polkadot_subsystem_bench::{approval, availability, configuration};
 use pyroscope::PyroscopeAgent;
 use pyroscope_pprofrs::{pprof_backend, PprofConfig};
 use serde::{Deserialize, Serialize};
 use std::path::Path;
 
-mod approval;
-mod availability;
-mod core;
 mod valgrind;
 
-const LOG_TARGET: &str = "subsystem-bench";
-
-fn le_100(s: &str) -> Result<usize, String> {
-	number_range(s, 0, 100)
-}
-
-fn le_5000(s: &str) -> Result<usize, String> {
-	number_range(s, 0, 5000)
-}
+const LOG_TARGET: &str = "subsystem-bench::cli";
 
 /// Supported test objectives
 #[derive(Debug, Clone, Parser, Serialize, Deserialize)]
 #[command(rename_all = "kebab-case")]
 pub enum TestObjective {
 	/// Benchmark availability recovery strategies.
-	DataAvailabilityRead(DataAvailabilityReadOptions),
+	DataAvailabilityRead(availability::DataAvailabilityReadOptions),
 	/// Benchmark availability and bitfield distribution.
 	DataAvailabilityWrite,
 	/// Benchmark the approval-voting and approval-distribution subsystems.
-	ApprovalVoting(ApprovalsOptions),
-	Unimplemented,
+	ApprovalVoting(approval::ApprovalsOptions),
 }
 
 impl std::fmt::Display for TestObjective {
@@ -73,39 +51,37 @@ impl std::fmt::Display for TestObjective {
 				Self::DataAvailabilityRead(_) => "DataAvailabilityRead",
 				Self::DataAvailabilityWrite => "DataAvailabilityWrite",
 				Self::ApprovalVoting(_) => "ApprovalVoting",
-				Self::Unimplemented => "Unimplemented",
 			}
 		)
 	}
 }
 
+/// The test input parameters
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct CliTestConfiguration {
+	/// Test Objective
+	pub objective: TestObjective,
+	/// Test Configuration
+	#[serde(flatten)]
+	pub test_config: configuration::TestConfiguration,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TestSequence {
+	#[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))]
+	test_configurations: Vec<CliTestConfiguration>,
+}
+
+impl TestSequence {
+	fn new_from_file(path: &Path) -> std::io::Result<TestSequence> {
+		let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8");
+		Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA"))
+	}
+}
+
 #[derive(Debug, Parser)]
 #[allow(missing_docs)]
 struct BenchCli {
-	#[arg(long, value_enum, ignore_case = true, default_value_t = NetworkEmulation::Ideal)]
-	/// The type of network to be emulated
-	pub network: NetworkEmulation,
-
-	#[clap(short, long)]
-	/// The bandwidth of emulated remote peers in KiB
-	pub peer_bandwidth: Option<usize>,
-
-	#[clap(short, long)]
-	/// The bandwidth of our node in KiB
-	pub bandwidth: Option<usize>,
-
-	#[clap(long, value_parser=le_100)]
-	/// Emulated peer connection ratio [0-100].
-	pub connectivity: Option<usize>,
-
-	#[clap(long, value_parser=le_5000)]
-	/// Mean remote peer latency in milliseconds [0-5000].
-	pub peer_mean_latency: Option<usize>,
-
-	#[clap(long, value_parser=le_5000)]
-	/// Remote peer latency standard deviation
-	pub peer_latency_std_dev: Option<f64>,
-
 	#[clap(long, default_value_t = false)]
 	/// Enable CPU Profiling with Pyroscope
 	pub profile: bool,
@@ -122,10 +98,6 @@ struct BenchCli {
 	/// Enable Cache Misses Profiling with Valgrind. Linux only, Valgrind must be in the PATH
 	pub cache_misses: bool,
 
-	#[clap(long, default_value_t = false)]
-	/// Shows the output in YAML format
-	pub yaml_output: bool,
-
 	#[arg(required = true)]
 	/// Path to the test sequence configuration file
 	pub path: String,
@@ -148,49 +120,60 @@ impl BenchCli {
 			None
 		};
 
-		let test_sequence = core::configuration::TestSequence::new_from_file(Path::new(&self.path))
+		let test_sequence = TestSequence::new_from_file(Path::new(&self.path))
 			.expect("File exists")
-			.into_vec();
+			.test_configurations;
 		let num_steps = test_sequence.len();
 		gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple());
-		for (index, test_config) in test_sequence.into_iter().enumerate() {
-			let benchmark_name = format!("{} #{} {}", &self.path, index + 1, test_config.objective);
-			gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),);
-			display_configuration(&test_config);
 
-			let usage = match test_config.objective {
-				TestObjective::DataAvailabilityRead(ref _opts) => {
-					let mut state = TestState::new(&test_config);
-					let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
+		for (index, CliTestConfiguration { objective, mut test_config }) in
+			test_sequence.into_iter().enumerate()
+		{
+			let benchmark_name = format!("{} #{} {}", &self.path, index + 1, objective);
+			gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),);
+			gum::info!(target: LOG_TARGET, "[{}] {}", format!("objective = {:?}", objective).green(), test_config);
+			test_config.generate_pov_sizes();
+
+			let usage = match objective {
+				TestObjective::DataAvailabilityRead(opts) => {
+					let mut state = availability::TestState::new(&test_config);
+					let (mut env, _protocol_config) = availability::prepare_test(
+						test_config,
+						&mut state,
+						availability::TestDataAvailability::Read(opts),
+						true,
+					);
 					env.runtime().block_on(availability::benchmark_availability_read(
 						&benchmark_name,
 						&mut env,
 						state,
 					))
 				},
+				TestObjective::DataAvailabilityWrite => {
+					let mut state = availability::TestState::new(&test_config);
+					let (mut env, _protocol_config) = availability::prepare_test(
+						test_config,
+						&mut state,
+						availability::TestDataAvailability::Write,
+						true,
+					);
+					env.runtime().block_on(availability::benchmark_availability_write(
+						&benchmark_name,
+						&mut env,
+						state,
+					))
+				},
 				TestObjective::ApprovalVoting(ref options) => {
 					let (mut env, state) =
-						approval::prepare_test(test_config.clone(), options.clone());
-					env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state))
-				},
-				TestObjective::DataAvailabilityWrite => {
-					let mut state = TestState::new(&test_config);
-					let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
-					env.runtime().block_on(availability::benchmark_availability_write(
+						approval::prepare_test(test_config.clone(), options.clone(), true);
+					env.runtime().block_on(approval::bench_approvals(
 						&benchmark_name,
 						&mut env,
 						state,
 					))
 				},
-				TestObjective::Unimplemented => todo!(),
 			};
-
-			let output = if self.yaml_output {
-				serde_yaml::to_string(&vec![usage])?
-			} else {
-				usage.to_string()
-			};
-			println!("{}", output);
+			println!("{}", usage);
 		}
 
 		if let Some(agent_running) = agent_running {

polkadot/node/subsystem-bench/src/lib/approval/helpers.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::core::configuration::TestAuthorities;
+use crate::configuration::TestAuthorities;
 use itertools::Itertools;
 use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick};
 use polkadot_node_network_protocol::{

polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs

@@ -18,15 +18,12 @@ use crate::{
 	approval::{
 		helpers::{generate_babe_epoch, generate_topology},
 		test_message::{MessagesBundle, TestMessageInfo},
-		ApprovalTestState, BlockTestData, GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET,
-		SLOT_DURATION_MILLIS,
+		ApprovalTestState, ApprovalsOptions, BlockTestData, GeneratedState,
+		BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, SLOT_DURATION_MILLIS,
 	},
-	core::{
-		configuration::{TestAuthorities, TestConfiguration},
-		mock::runtime_api::session_info_for_peers,
-		NODE_UNDER_TEST,
-	},
-	ApprovalsOptions, TestObjective,
+	configuration::{TestAuthorities, TestConfiguration},
+	mock::runtime_api::session_info_for_peers,
+	NODE_UNDER_TEST,
 };
 use futures::SinkExt;
 use itertools::Itertools;
@@ -132,11 +129,7 @@ impl PeerMessagesGenerator {
 		options: &ApprovalsOptions,
 	) -> String {
 		let mut fingerprint = options.fingerprint();
-		let mut exclude_objective = configuration.clone();
-		// The objective contains the full content of `ApprovalOptions`, we don't want to put all of
-		// that in fingerprint, so execlute it because we add it manually see above.
-		exclude_objective.objective = TestObjective::Unimplemented;
-		let configuration_bytes = bincode::serialize(&exclude_objective).unwrap();
+		let configuration_bytes = bincode::serialize(&configuration).unwrap();
 		fingerprint.extend(configuration_bytes);
 		let mut sha1 = sha1::Sha1::new();
 		sha1.update(fingerprint);

polkadot/node/subsystem-bench/src/lib/approval/mod.rs

@@ -24,25 +24,21 @@ use crate::{
 		mock_chain_selection::MockChainSelection,
 		test_message::{MessagesBundle, TestMessageInfo},
 	},
-	core::{
-		configuration::TestAuthorities,
-		environment::{
-			BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT,
-		},
-		mock::{
-			chain_api::{ChainApiState, MockChainApi},
-			dummy_builder,
-			network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx},
-			runtime_api::MockRuntimeApi,
-			AlwaysSupportsParachains, TestSyncOracle,
-		},
-		network::{
-			new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface,
-			NetworkInterfaceReceiver,
-		},
-		NODE_UNDER_TEST,
+	configuration::{TestAuthorities, TestConfiguration},
+	dummy_builder,
+	environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT},
+	mock::{
+		chain_api::{ChainApiState, MockChainApi},
+		network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx},
+		runtime_api::MockRuntimeApi,
+		AlwaysSupportsParachains, TestSyncOracle,
 	},
-	TestConfiguration,
+	network::{
+		new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface,
+		NetworkInterfaceReceiver,
+	},
+	usage::BenchmarkUsage,
+	NODE_UNDER_TEST,
 };
 use colored::Colorize;
 use futures::channel::oneshot;
@@ -472,11 +468,9 @@ impl ApprovalTestState {
 impl HandleNetworkMessage for ApprovalTestState {
 	fn handle(
 		&self,
-		_message: crate::core::network::NetworkMessage,
-		_node_sender: &mut futures::channel::mpsc::UnboundedSender<
-			crate::core::network::NetworkMessage,
-		>,
-	) -> Option<crate::core::network::NetworkMessage> {
+		_message: crate::network::NetworkMessage,
+		_node_sender: &mut futures::channel::mpsc::UnboundedSender<crate::network::NetworkMessage>,
+	) -> Option<crate::network::NetworkMessage> {
 		self.total_sent_messages_from_node
 			.as_ref()
 			.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
@@ -841,8 +835,14 @@ fn build_overseer(
 pub fn prepare_test(
 	config: TestConfiguration,
 	options: ApprovalsOptions,
+	with_prometheus_endpoint: bool,
 ) -> (TestEnvironment, ApprovalTestState) {
-	prepare_test_inner(config, TestEnvironmentDependencies::default(), options)
+	prepare_test_inner(
+		config,
+		TestEnvironmentDependencies::default(),
+		options,
+		with_prometheus_endpoint,
+	)
 }
 
 /// Build the test environment for an Approval benchmark.
@@ -850,6 +850,7 @@ fn prepare_test_inner(
 	config: TestConfiguration,
 	dependencies: TestEnvironmentDependencies,
 	options: ApprovalsOptions,
+	with_prometheus_endpoint: bool,
 ) -> (TestEnvironment, ApprovalTestState) {
 	gum::info!("Prepare test state");
 	let state = ApprovalTestState::new(&config, options, &dependencies);
@@ -878,6 +879,7 @@ fn prepare_test_inner(
 			overseer,
 			overseer_handle,
 			state.test_authorities.clone(),
+			with_prometheus_endpoint,
 		),
 		state,
 	)

polkadot/node/subsystem-bench/src/lib/approval/test_message.rs

@@ -15,9 +15,8 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-	approval::{BlockTestData, CandidateTestData},
-	core::configuration::TestAuthorities,
-	ApprovalsOptions,
+	approval::{ApprovalsOptions, BlockTestData, CandidateTestData},
+	configuration::TestAuthorities,
 };
 use itertools::Itertools;
 use parity_scale_codec::{Decode, Encode};

polkadot/node/subsystem-bench/src/lib/availability/av_store_helpers.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::core::{environment::TestEnvironmentDependencies, mock::TestSyncOracle};
+use crate::{environment::TestEnvironmentDependencies, mock::TestSyncOracle};
 use polkadot_node_core_av_store::{AvailabilityStoreSubsystem, Config};
 use polkadot_node_metrics::metrics::Metrics;
 use polkadot_node_subsystem_util::database::Database;

polkadot/node/subsystem-bench/src/lib/availability/mod.rs

@@ -15,22 +15,18 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-	core::{
-		configuration::TestConfiguration,
-		environment::{BenchmarkUsage, TestEnvironmentDependencies},
-		mock::{
-			av_store,
-			av_store::MockAvailabilityStore,
-			chain_api::{ChainApiState, MockChainApi},
-			dummy_builder,
-			network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx},
-			runtime_api,
-			runtime_api::MockRuntimeApi,
-			AlwaysSupportsParachains,
-		},
-		network::new_network,
+	configuration::TestConfiguration,
+	dummy_builder,
+	environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH},
+	mock::{
+		av_store::{self, MockAvailabilityStore},
+		chain_api::{ChainApiState, MockChainApi},
+		network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx},
+		runtime_api::{self, MockRuntimeApi},
+		AlwaysSupportsParachains,
 	},
-	TestEnvironment, TestObjective, GENESIS_HASH,
+	network::new_network,
+	usage::BenchmarkUsage,
 };
 use av_store::NetworkAvailabilityState;
 use av_store_helpers::new_av_store;
@@ -73,14 +69,30 @@ use sc_network::{
 	PeerId,
 };
 use sc_service::SpawnTaskHandle;
+use serde::{Deserialize, Serialize};
 use sp_core::H256;
 use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant};
 
 mod av_store_helpers;
-pub(crate) mod cli;
 
 const LOG_TARGET: &str = "subsystem-bench::availability";
 
+#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)]
+#[clap(rename_all = "kebab-case")]
+#[allow(missing_docs)]
+pub struct DataAvailabilityReadOptions {
+	#[clap(short, long, default_value_t = false)]
+	/// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as
+	/// we don't need to re-construct from chunks. Tipically this is only faster if nodes have
+	/// enough bandwidth.
+	pub fetch_from_backers: bool,
+}
+
+pub enum TestDataAvailability {
+	Read(DataAvailabilityReadOptions),
+	Write,
+}
+
 fn build_overseer_for_availability_read(
 	spawn_task_handle: SpawnTaskHandle,
 	runtime_api: MockRuntimeApi,
@@ -141,14 +153,24 @@ fn build_overseer_for_availability_write(
 pub fn prepare_test(
 	config: TestConfiguration,
 	state: &mut TestState,
+	mode: TestDataAvailability,
+	with_prometheus_endpoint: bool,
 ) -> (TestEnvironment, Vec<ProtocolConfig>) {
-	prepare_test_inner(config, state, TestEnvironmentDependencies::default())
+	prepare_test_inner(
+		config,
+		state,
+		mode,
+		TestEnvironmentDependencies::default(),
+		with_prometheus_endpoint,
+	)
 }
 
 fn prepare_test_inner(
 	config: TestConfiguration,
 	state: &mut TestState,
+	mode: TestDataAvailability,
 	dependencies: TestEnvironmentDependencies,
+	with_prometheus_endpoint: bool,
 ) -> (TestEnvironment, Vec<ProtocolConfig>) {
 	// Generate test authorities.
 	let test_authorities = config.generate_authorities();
@@ -216,8 +238,8 @@ fn prepare_test_inner(
 	let network_bridge_rx =
 		network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone()));
 
-	let (overseer, overseer_handle) = match &state.config().objective {
-		TestObjective::DataAvailabilityRead(options) => {
+	let (overseer, overseer_handle) = match &mode {
+		TestDataAvailability::Read(options) => {
 			let use_fast_path = options.fetch_from_backers;
 
 			let subsystem = if use_fast_path {
@@ -247,7 +269,7 @@ fn prepare_test_inner(
 				&dependencies,
 			)
 		},
-		TestObjective::DataAvailabilityWrite => {
+		TestDataAvailability::Write => {
 			let availability_distribution = AvailabilityDistributionSubsystem::new(
 				test_authorities.keyring.keystore(),
 				IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver },
@@ -284,9 +306,6 @@ fn prepare_test_inner(
 				&dependencies,
 			)
 		},
-		_ => {
-			unimplemented!("Invalid test objective")
-		},
 	};
 
 	(
@@ -297,6 +316,7 @@ fn prepare_test_inner(
 			overseer,
 			overseer_handle,
 			test_authorities,
+			with_prometheus_endpoint,
 		),
 		req_cfgs,
 	)
@@ -326,10 +346,6 @@ pub struct TestState {
 }
 
 impl TestState {
-	fn config(&self) -> &TestConfiguration {
-		&self.config
-	}
-
 	pub fn next_candidate(&mut self) -> Option<CandidateReceipt> {
 		let candidate = self.candidates.next();
 		let candidate_hash = candidate.as_ref().unwrap().hash();

polkadot/node/subsystem-bench/src/lib/configuration.rs

@@ -16,7 +16,7 @@
 
 //! Test configuration definition and helpers.
 
-use crate::{core::keyring::Keyring, TestObjective};
+use crate::keyring::Keyring;
 use itertools::Itertools;
 use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId};
 use rand::thread_rng;
@@ -24,17 +24,7 @@ use rand_distr::{Distribution, Normal, Uniform};
 use sc_network::PeerId;
 use serde::{Deserialize, Serialize};
 use sp_consensus_babe::AuthorityId;
-use std::{collections::HashMap, path::Path};
-
-pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize {
-	random_uniform_sample(min_pov_size, max_pov_size)
-}
-
-fn random_uniform_sample<T: Into<usize> + From<usize>>(min_value: T, max_value: T) -> T {
-	Uniform::from(min_value.into()..=max_value.into())
-		.sample(&mut thread_rng())
-		.into()
-}
+use std::collections::HashMap;
 
 /// Peer networking latency configuration.
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -87,8 +77,6 @@ fn default_no_show_slots() -> usize {
 /// The test input parameters
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct TestConfiguration {
-	/// The test objective
-	pub objective: TestObjective,
 	/// Number of validators
 	pub n_validators: usize,
 	/// Number of cores
@@ -115,7 +103,7 @@ pub struct TestConfiguration {
 	pub max_pov_size: usize,
 	/// Randomly sampled pov_sizes
 	#[serde(skip)]
-	pov_sizes: Vec<usize>,
+	pub pov_sizes: Vec<usize>,
 	/// The amount of bandiwdth remote validators have.
 	#[serde(default = "default_bandwidth")]
 	pub peer_bandwidth: usize,
@@ -133,56 +121,32 @@ pub struct TestConfiguration {
 	pub num_blocks: usize,
 }
 
-fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec<usize> {
-	(0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect()
-}
-
-#[derive(Serialize, Deserialize)]
-pub struct TestSequence {
-	#[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))]
-	test_configurations: Vec<TestConfiguration>,
-}
-
-impl TestSequence {
-	pub fn into_vec(self) -> Vec<TestConfiguration> {
-		self.test_configurations
-			.into_iter()
-			.map(|mut config| {
-				config.pov_sizes =
-					generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size);
-				config
-			})
-			.collect()
+impl Default for TestConfiguration {
+	fn default() -> Self {
+		Self {
+			n_validators: Default::default(),
+			n_cores: Default::default(),
+			needed_approvals: default_needed_approvals(),
+			zeroth_delay_tranche_width: default_zeroth_delay_tranche_width(),
+			relay_vrf_modulo_samples: default_relay_vrf_modulo_samples(),
+			n_delay_tranches: default_n_delay_tranches(),
+			no_show_slots: default_no_show_slots(),
+			max_validators_per_core: default_backing_group_size(),
+			min_pov_size: default_pov_size(),
+			max_pov_size: default_pov_size(),
+			pov_sizes: Default::default(),
+			peer_bandwidth: default_bandwidth(),
+			bandwidth: default_bandwidth(),
+			latency: Default::default(),
+			connectivity: default_connectivity(),
+			num_blocks: Default::default(),
+		}
 	}
 }
 
-impl TestSequence {
-	pub fn new_from_file(path: &Path) -> std::io::Result<TestSequence> {
-		let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8");
-		Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA"))
-	}
-}
-
-/// Helper struct for authority related state.
-#[derive(Clone)]
-pub struct TestAuthorities {
-	pub keyring: Keyring,
-	pub validator_public: Vec<ValidatorId>,
-	pub validator_authority_id: Vec<AuthorityDiscoveryId>,
-	pub validator_babe_id: Vec<AuthorityId>,
-	pub validator_assignment_id: Vec<AssignmentId>,
-	pub key_seeds: Vec<String>,
-	pub peer_ids: Vec<PeerId>,
-	pub peer_id_to_authority: HashMap<PeerId, AuthorityDiscoveryId>,
-}
-
 impl TestConfiguration {
-	#[allow(unused)]
-	pub fn write_to_disk(&self) {
-		// Serialize a slice of configurations
-		let yaml = serde_yaml::to_string(&TestSequence { test_configurations: vec![self.clone()] })
-			.unwrap();
-		std::fs::write("last_test.yaml", yaml).unwrap();
+	pub fn generate_pov_sizes(&mut self) {
+		self.pov_sizes = generate_pov_sizes(self.n_cores, self.min_pov_size, self.max_pov_size);
 	}
 
 	pub fn pov_sizes(&self) -> &[usize] {
@@ -239,6 +203,33 @@ impl TestConfiguration {
 	}
 }
 
+fn random_uniform_sample<T: Into<usize> + From<usize>>(min_value: T, max_value: T) -> T {
+	Uniform::from(min_value.into()..=max_value.into())
+		.sample(&mut thread_rng())
+		.into()
+}
+
+fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize {
+	random_uniform_sample(min_pov_size, max_pov_size)
+}
+
+fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec<usize> {
+	(0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect()
+}
+
+/// Helper struct for authority related state.
+#[derive(Clone)]
+pub struct TestAuthorities {
+	pub keyring: Keyring,
+	pub validator_public: Vec<ValidatorId>,
+	pub validator_authority_id: Vec<AuthorityDiscoveryId>,
+	pub validator_babe_id: Vec<AuthorityId>,
+	pub validator_assignment_id: Vec<AssignmentId>,
+	pub key_seeds: Vec<String>,
+	pub peer_ids: Vec<PeerId>,
+	pub peer_id_to_authority: HashMap<PeerId, AuthorityDiscoveryId>,
+}
+
 /// Sample latency (in milliseconds) from a normal distribution with parameters
 /// specified in `maybe_peer_latency`.
 pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> usize {

polkadot/node/subsystem-bench/src/lib/display.rs

@@ -19,7 +19,7 @@
 //!
 //! Currently histogram buckets are skipped.
 
-use crate::{TestConfiguration, LOG_TARGET};
+use crate::configuration::TestConfiguration;
 use colored::Colorize;
 use prometheus::{
 	proto::{MetricFamily, MetricType},
@@ -27,6 +27,8 @@ use prometheus::{
 };
 use std::fmt::Display;
 
+const LOG_TARGET: &str = "subsystem-bench::display";
+
 #[derive(Default, Debug)]
 pub struct MetricCollection(Vec<TestMetric>);
 
@@ -85,6 +87,7 @@ impl Display for MetricCollection {
 		Ok(())
 	}
 }
+
 #[derive(Debug, Clone)]
 pub struct TestMetric {
 	name: String,
@@ -184,15 +187,16 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection {
 	test_metrics.into()
 }
 
-pub fn display_configuration(test_config: &TestConfiguration) {
-	gum::info!(
-		"[{}] {}, {}, {}, {}, {}",
-		format!("objective = {:?}", test_config.objective).green(),
-		format!("n_validators = {}", test_config.n_validators).blue(),
-		format!("n_cores = {}", test_config.n_cores).blue(),
-		format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size)
-			.bright_black(),
-		format!("connectivity = {}", test_config.connectivity).bright_black(),
-		format!("latency = {:?}", test_config.latency).bright_black(),
-	);
+impl Display for TestConfiguration {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		write!(
+			f,
+			"{}, {}, {}, {}, {}",
+			format!("n_validators = {}", self.n_validators).blue(),
+			format!("n_cores = {}", self.n_cores).blue(),
+			format!("pov_size = {} - {}", self.min_pov_size, self.max_pov_size).bright_black(),
+			format!("connectivity = {}", self.connectivity).bright_black(),
+			format!("latency = {:?}", self.latency).bright_black(),
+		)
+	}
 }

polkadot/node/subsystem-bench/src/lib/environment.rs

@@ -17,13 +17,11 @@
 //! Test environment implementation
 
 use crate::{
-	core::{
-		configuration::TestAuthorities, mock::AlwaysSupportsParachains,
-		network::NetworkEmulatorHandle,
-	},
-	TestConfiguration,
+	configuration::{TestAuthorities, TestConfiguration},
+	mock::AlwaysSupportsParachains,
+	network::NetworkEmulatorHandle,
+	usage::{BenchmarkUsage, ResourceUsage},
 };
-use colored::Colorize;
 use core::time::Duration;
 use futures::{Future, FutureExt};
 use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt};
@@ -33,7 +31,6 @@ use polkadot_node_subsystem_util::metrics::prometheus::{
 };
 use polkadot_overseer::{BlockInfo, Handle as OverseerHandle};
 use sc_service::{SpawnTaskHandle, TaskManager};
-use serde::{Deserialize, Serialize};
 use std::net::{Ipv4Addr, SocketAddr};
 use tokio::runtime::Handle;
 
@@ -204,6 +201,7 @@ impl TestEnvironment {
 		overseer: Overseer<SpawnGlue<SpawnTaskHandle>, AlwaysSupportsParachains>,
 		overseer_handle: OverseerHandle,
 		authorities: TestAuthorities,
+		with_prometheus_endpoint: bool,
 	) -> Self {
 		let metrics = TestEnvironmentMetrics::new(&dependencies.registry)
 			.expect("Metrics need to be registered");
@@ -211,19 +209,21 @@ impl TestEnvironment {
 		let spawn_handle = dependencies.task_manager.spawn_handle();
 		spawn_handle.spawn_blocking("overseer", "overseer", overseer.run().boxed());
 
-		let registry_clone = dependencies.registry.clone();
-		dependencies.task_manager.spawn_handle().spawn_blocking(
-			"prometheus",
-			"test-environment",
-			async move {
-				prometheus_endpoint::init_prometheus(
-					SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999),
-					registry_clone,
-				)
-				.await
-				.unwrap();
-			},
-		);
+		if with_prometheus_endpoint {
+			let registry_clone = dependencies.registry.clone();
+			dependencies.task_manager.spawn_handle().spawn_blocking(
+				"prometheus",
+				"test-environment",
+				async move {
+					prometheus_endpoint::init_prometheus(
+						SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999),
+						registry_clone,
+					)
+					.await
+					.unwrap();
+				},
+			);
+		}
 
 		TestEnvironment {
 			runtime_handle: dependencies.runtime.handle().clone(),
@@ -411,41 +411,3 @@ impl TestEnvironment {
 		usage
 	}
 }
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct BenchmarkUsage {
-	benchmark_name: String,
-	network_usage: Vec<ResourceUsage>,
-	cpu_usage: Vec<ResourceUsage>,
-}
-
-impl std::fmt::Display for BenchmarkUsage {
-	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-		write!(
-			f,
-			"\n{}\n\n{}\n{}\n\n{}\n{}\n",
-			self.benchmark_name.purple(),
-			format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(),
-			self.network_usage
-				.iter()
-				.map(|v| v.to_string())
-				.collect::<Vec<String>>()
-				.join("\n"),
-			format!("{:<32}{:>12}{:>12}", "CPU usage in seconds", "total", "per block").blue(),
-			self.cpu_usage.iter().map(|v| v.to_string()).collect::<Vec<String>>().join("\n")
-		)
-	}
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ResourceUsage {
-	resource_name: String,
-	total: f64,
-	per_block: f64,
-}
-
-impl std::fmt::Display for ResourceUsage {
-	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-		write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block)
-	}
-}

polkadot/node/subsystem-bench/src/lib/lib.rs

@@ -15,11 +15,14 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 // The validator index that represent the node that is under test.
-pub(crate) const NODE_UNDER_TEST: u32 = 0;
+pub const NODE_UNDER_TEST: u32 = 0;
 
-pub(crate) mod configuration;
+pub mod approval;
+pub mod availability;
+pub mod configuration;
 pub(crate) mod display;
 pub(crate) mod environment;
 pub(crate) mod keyring;
 pub(crate) mod mock;
 pub(crate) mod network;
+pub mod usage;

polkadot/node/subsystem-bench/src/lib/mock/av_store.rs

@@ -16,7 +16,7 @@
 
 //! A generic av store subsystem mockup suitable to be used in benchmarks.
 
-use crate::core::network::{HandleNetworkMessage, NetworkMessage};
+use crate::network::{HandleNetworkMessage, NetworkMessage};
 use futures::{channel::oneshot, FutureExt};
 use parity_scale_codec::Encode;
 use polkadot_node_network_protocol::request_response::{

polkadot/node/subsystem-bench/src/lib/mock/mod.rs

@@ -34,9 +34,10 @@ impl HeadSupportsParachains for AlwaysSupportsParachains {
 }
 
 // An orchestra with dummy subsystems
+#[macro_export]
 macro_rules! dummy_builder {
 	($spawn_task_handle: ident, $metrics: ident) => {{
-		use $crate::core::mock::dummy::*;
+		use $crate::mock::dummy::*;
 
 		// Initialize a mock overseer.
 		// All subsystem except approval_voting and approval_distribution are mock subsystems.
@@ -72,7 +73,6 @@ macro_rules! dummy_builder {
 			.spawner(SpawnGlue($spawn_task_handle))
 	}};
 }
-pub(crate) use dummy_builder;
 
 #[derive(Clone)]
 pub struct TestSyncOracle {}

polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs

@@ -17,7 +17,7 @@
 //! Mocked `network-bridge` subsystems that uses a `NetworkInterface` to access
 //! the emulated network.
 
-use crate::core::{
+use crate::{
 	configuration::TestAuthorities,
 	network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage, RequestExt},
 };

polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs

@@ -16,7 +16,7 @@
 
 //! A generic runtime api subsystem mockup suitable to be used in benchmarks.
 
-use crate::core::configuration::{TestAuthorities, TestConfiguration};
+use crate::configuration::{TestAuthorities, TestConfiguration};
 use bitvec::prelude::BitVec;
 use futures::FutureExt;
 use itertools::Itertools;

polkadot/node/subsystem-bench/src/lib/network.rs

@@ -33,7 +33,7 @@
 //               |
 //     Subsystems under test
 
-use crate::core::{
+use crate::{
 	configuration::{random_latency, TestAuthorities, TestConfiguration},
 	environment::TestEnvironmentDependencies,
 	NODE_UNDER_TEST,

polkadot/node/subsystem-bench/src/lib/usage.rs

@@ -0,0 +1,146 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test usage implementation
+
+use colored::Colorize;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BenchmarkUsage {
+	pub benchmark_name: String,
+	pub network_usage: Vec<ResourceUsage>,
+	pub cpu_usage: Vec<ResourceUsage>,
+}
+
+impl std::fmt::Display for BenchmarkUsage {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(
+			f,
+			"\n{}\n\n{}\n{}\n\n{}\n{}\n",
+			self.benchmark_name.purple(),
+			format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(),
+			self.network_usage
+				.iter()
+				.map(|v| v.to_string())
+				.collect::<Vec<String>>()
+				.join("\n"),
+			format!("{:<32}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(),
+			self.cpu_usage.iter().map(|v| v.to_string()).collect::<Vec<String>>().join("\n")
+		)
+	}
+}
+
+impl BenchmarkUsage {
+	pub fn average(usages: &[Self]) -> Self {
+		let all_network_usages: Vec<&ResourceUsage> =
+			usages.iter().flat_map(|v| &v.network_usage).collect();
+		let all_cpu_usage: Vec<&ResourceUsage> = usages.iter().flat_map(|v| &v.cpu_usage).collect();
+
+		Self {
+			benchmark_name: usages.first().map(|v| v.benchmark_name.clone()).unwrap_or_default(),
+			network_usage: ResourceUsage::average_by_resource_name(&all_network_usages),
+			cpu_usage: ResourceUsage::average_by_resource_name(&all_cpu_usage),
+		}
+	}
+
+	pub fn check_network_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
+		check_usage(&self.benchmark_name, &self.network_usage, checks)
+	}
+
+	pub fn check_cpu_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
+		check_usage(&self.benchmark_name, &self.cpu_usage, checks)
+	}
+
+	pub fn cpu_usage_diff(&self, other: &Self, resource_name: &str) -> Option<f64> {
+		let self_res = self.cpu_usage.iter().find(|v| v.resource_name == resource_name);
+		let other_res = other.cpu_usage.iter().find(|v| v.resource_name == resource_name);
+
+		match (self_res, other_res) {
+			(Some(self_res), Some(other_res)) => Some(self_res.diff(other_res)),
+			_ => None,
+		}
+	}
+}
+
+fn check_usage(
+	benchmark_name: &str,
+	usage: &[ResourceUsage],
+	checks: &[ResourceUsageCheck],
+) -> Vec<String> {
+	checks
+		.iter()
+		.filter_map(|check| {
+			check_resource_usage(usage, check)
+				.map(|message| format!("{}: {}", benchmark_name, message))
+		})
+		.collect()
+}
+
+fn check_resource_usage(
+	usage: &[ResourceUsage],
+	(resource_name, base, precision): &ResourceUsageCheck,
+) -> Option<String> {
+	if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) {
+		let diff = (base - usage.per_block).abs() / base;
+		if diff < *precision {
+			None
+		} else {
+			Some(format!(
+				"The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {}",
+				resource_name, base, precision, usage.per_block
+			))
+		}
+	} else {
+		Some(format!("The resource `{}` is not found", resource_name))
+	}
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ResourceUsage {
+	pub resource_name: String,
+	pub total: f64,
+	pub per_block: f64,
+}
+
+impl std::fmt::Display for ResourceUsage {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block)
+	}
+}
+
+impl ResourceUsage {
+	fn average_by_resource_name(usages: &[&Self]) -> Vec<Self> {
+		let mut by_name: HashMap<String, Vec<&Self>> = Default::default();
+		for usage in usages {
+			by_name.entry(usage.resource_name.clone()).or_default().push(usage);
+		}
+		let mut average = vec![];
+		for (resource_name, values) in by_name {
+			let total = values.iter().map(|v| v.total).sum::<f64>() / values.len() as f64;
+			let per_block = values.iter().map(|v| v.per_block).sum::<f64>() / values.len() as f64;
+			average.push(Self { resource_name, total, per_block });
+		}
+		average
+	}
+
+	fn diff(&self, other: &Self) -> f64 {
+		(self.per_block - other.per_block).abs() / self.per_block
+	}
+}
+
+type ResourceUsageCheck<'a> = (&'a str, f64, f64);