subsystem-bench: add regression tests for availability read and write (#3311)

### What's been done
- `subsystem-bench` has been split into two parts: a cli benchmark
runner and a library.
- The cli runner is quite simple. It just allows us to run `.yaml` based
test sequences. Now it should only be used to run benchmarks during
development.
- The library is used in the cli runner and in regression tests. Some
code is changed to make the library independent of the runner.
- Added first regression tests for availability read and write that
replicate existing test sequences.

### How we run regression tests
- Regression tests are simply rust integration tests without the
harnesses.
- They should only be compiled under the `subsystem-benchmarks` feature
to prevent them from running with other tests.
- This doesn't work when running tests with `nextest` in CI, so
additional filters have been added to the `nextest` runs.
- Each benchmark run takes a different time in the beginning, so we
"warm up" the tests until their CPU usage differs by only 1%.
- After the warm-up, we run the benchmarks a few more times and compare
the average with the exception using a precision.

### What is still wrong?
- I haven't managed to set up approval voting tests. The spread of their
results is too large and can't be narrowed down in a reasonable amount
of time in the warm-up phase.
- The tests start an unconfigurable prometheus endpoint inside, which
causes errors because they use the same 9999 port. I disable it with a
flag, but I think it's better to extract the endpoint launching outside
the test, as we already do with `valgrind` and `pyroscope`. But we still
use `prometheus` inside the tests.

### Future work
* https://github.com/paritytech/polkadot-sdk/issues/3528
* https://github.com/paritytech/polkadot-sdk/issues/3529
* https://github.com/paritytech/polkadot-sdk/issues/3530
* https://github.com/paritytech/polkadot-sdk/issues/3531

---------

Co-authored-by: Alexander Samusev <41779041+alvicsam@users.noreply.github.com>

polkadot/node/subsystem-bench/src/lib/usage.rs

@@ -0,0 +1,146 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test usage implementation
+
+use colored::Colorize;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BenchmarkUsage {
+	pub benchmark_name: String,
+	pub network_usage: Vec<ResourceUsage>,
+	pub cpu_usage: Vec<ResourceUsage>,
+}
+
+impl std::fmt::Display for BenchmarkUsage {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(
+			f,
+			"\n{}\n\n{}\n{}\n\n{}\n{}\n",
+			self.benchmark_name.purple(),
+			format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(),
+			self.network_usage
+				.iter()
+				.map(|v| v.to_string())
+				.collect::<Vec<String>>()
+				.join("\n"),
+			format!("{:<32}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(),
+			self.cpu_usage.iter().map(|v| v.to_string()).collect::<Vec<String>>().join("\n")
+		)
+	}
+}
+
+impl BenchmarkUsage {
+	pub fn average(usages: &[Self]) -> Self {
+		let all_network_usages: Vec<&ResourceUsage> =
+			usages.iter().flat_map(|v| &v.network_usage).collect();
+		let all_cpu_usage: Vec<&ResourceUsage> = usages.iter().flat_map(|v| &v.cpu_usage).collect();
+
+		Self {
+			benchmark_name: usages.first().map(|v| v.benchmark_name.clone()).unwrap_or_default(),
+			network_usage: ResourceUsage::average_by_resource_name(&all_network_usages),
+			cpu_usage: ResourceUsage::average_by_resource_name(&all_cpu_usage),
+		}
+	}
+
+	pub fn check_network_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
+		check_usage(&self.benchmark_name, &self.network_usage, checks)
+	}
+
+	pub fn check_cpu_usage(&self, checks: &[ResourceUsageCheck]) -> Vec<String> {
+		check_usage(&self.benchmark_name, &self.cpu_usage, checks)
+	}
+
+	pub fn cpu_usage_diff(&self, other: &Self, resource_name: &str) -> Option<f64> {
+		let self_res = self.cpu_usage.iter().find(|v| v.resource_name == resource_name);
+		let other_res = other.cpu_usage.iter().find(|v| v.resource_name == resource_name);
+
+		match (self_res, other_res) {
+			(Some(self_res), Some(other_res)) => Some(self_res.diff(other_res)),
+			_ => None,
+		}
+	}
+}
+
+fn check_usage(
+	benchmark_name: &str,
+	usage: &[ResourceUsage],
+	checks: &[ResourceUsageCheck],
+) -> Vec<String> {
+	checks
+		.iter()
+		.filter_map(|check| {
+			check_resource_usage(usage, check)
+				.map(|message| format!("{}: {}", benchmark_name, message))
+		})
+		.collect()
+}
+
+fn check_resource_usage(
+	usage: &[ResourceUsage],
+	(resource_name, base, precision): &ResourceUsageCheck,
+) -> Option<String> {
+	if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) {
+		let diff = (base - usage.per_block).abs() / base;
+		if diff < *precision {
+			None
+		} else {
+			Some(format!(
+				"The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {}",
+				resource_name, base, precision, usage.per_block
+			))
+		}
+	} else {
+		Some(format!("The resource `{}` is not found", resource_name))
+	}
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ResourceUsage {
+	pub resource_name: String,
+	pub total: f64,
+	pub per_block: f64,
+}
+
+impl std::fmt::Display for ResourceUsage {
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+		write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block)
+	}
+}
+
+impl ResourceUsage {
+	fn average_by_resource_name(usages: &[&Self]) -> Vec<Self> {
+		let mut by_name: HashMap<String, Vec<&Self>> = Default::default();
+		for usage in usages {
+			by_name.entry(usage.resource_name.clone()).or_default().push(usage);
+		}
+		let mut average = vec![];
+		for (resource_name, values) in by_name {
+			let total = values.iter().map(|v| v.total).sum::<f64>() / values.len() as f64;
+			let per_block = values.iter().map(|v| v.per_block).sum::<f64>() / values.len() as f64;
+			average.push(Self { resource_name, total, per_block });
+		}
+		average
+	}
+
+	fn diff(&self, other: &Self) -> f64 {
+		(self.per_block - other.per_block).abs() / self.per_block
+	}
+}
+
+type ResourceUsageCheck<'a> = (&'a str, f64, f64);