cli: introduce host-perf-check command (#4342)

polkadot/Cargo.lock

@@ -5865,9 +5865,11 @@ dependencies = [
  "futures 0.3.18",
  "log",
  "polkadot-node-core-pvf",
+ "polkadot-performance-test",
  "polkadot-service",
  "sc-cli",
  "sc-service",
+ "sc-tracing",
  "sp-core",
  "sp-trie",
  "structopt",
@@ -6540,6 +6542,20 @@ dependencies = [
  "sp-std",
 ]
 
+[[package]]
+name = "polkadot-performance-test"
+version = "0.9.13"
+dependencies = [
+ "env_logger 0.9.0",
+ "kusama-runtime",
+ "log",
+ "polkadot-erasure-coding",
+ "polkadot-node-core-pvf",
+ "polkadot-node-primitives",
+ "quote",
+ "thiserror",
+]
+
 [[package]]
 name = "polkadot-primitives"
 version = "0.9.13"

polkadot/Cargo.toml

@@ -84,6 +84,7 @@ members = [
 	"node/metrics",
 	"node/metered-channel",
 	"node/test/client",
+	"node/test/performance-test",
 	"node/test/service",
 	"node/test/polkadot-simnet/common",
 	"node/test/polkadot-simnet/node",

polkadot/cli/Cargo.toml

@@ -21,12 +21,14 @@ futures = "0.3.17"
 
 service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true }
 polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true }
+polkadot-performance-test = { path = "../node/test/performance-test", optional = true }
 
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
+sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 
 # this crate is used only to enable `trie-memory-tracker` feature
 # see https://github.com/paritytech/substrate/pull/6745
@@ -43,9 +45,11 @@ cli = [
 	"structopt",
 	"sc-cli",
 	"sc-service",
+	"sc-tracing",
 	"frame-benchmarking-cli",
 	"try-runtime-cli",
 	"polkadot-node-core-pvf",
+	"polkadot-performance-test",
 ]
 runtime-benchmarks = [ "service/runtime-benchmarks" ]
 trie-memory-tracker = [ "sp-trie/memory-tracker" ]

polkadot/cli/build.rs

@@ -1,4 +1,4 @@
-// Copyright 2015-2020 Parity Technologies (UK) Ltd.
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
 // This file is part of Polkadot.
 
 // Polkadot is free software: you can redistribute it and/or modify
@@ -15,5 +15,8 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 fn main() {
+	if let Ok(profile) = std::env::var("PROFILE") {
+		println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
+	}
 	substrate_build_script_utils::generate_cargo_keys();
 }

polkadot/cli/src/cli.rs

@@ -54,6 +54,10 @@ pub enum Subcommand {
 	#[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
+	/// Runs performance checks such as PVF compilation in order to measure machine
+	/// capabilities of running a validator.
+	HostPerfCheck,
+
 	/// Try some command against runtime state.
 	#[cfg(feature = "try-runtime")]
 	TryRuntime(try_runtime_cli::TryRuntimeCmd),

polkadot/cli/src/command.rs

@@ -21,20 +21,8 @@ use sc_cli::{Role, RuntimeVersion, SubstrateCli};
 use service::{self, IdentifyVariant};
 use sp_core::crypto::Ss58AddressFormatRegistry;
 
-#[derive(thiserror::Error, Debug)]
-pub enum Error {
-	#[error(transparent)]
-	PolkadotService(#[from] service::Error),
-
-	#[error(transparent)]
-	SubstrateCli(#[from] sc_cli::Error),
-
-	#[error(transparent)]
-	SubstrateService(#[from] sc_service::Error),
-
-	#[error("Other: {0}")]
-	Other(String),
-}
+pub use crate::error::Error;
+pub use polkadot_performance_test::PerfCheckError;
 
 impl std::convert::From<String> for Error {
 	fn from(s: String) -> Self {
@@ -215,6 +203,20 @@ fn ensure_dev(spec: &Box<dyn service::ChainSpec>) -> std::result::Result<(), Str
 	}
 }
 
+/// Runs performance checks.
+/// Should only be used in release build since the check would take too much time otherwise.
+fn host_perf_check() -> Result<()> {
+	#[cfg(not(build_type = "release"))]
+	{
+		Err(PerfCheckError::WrongBuildType.into())
+	}
+	#[cfg(build_type = "release")]
+	{
+		crate::host_perf_check::host_perf_check()?;
+		Ok(())
+	}
+}
+
 /// Launch a node, accepting arguments just like a regular node,
 /// accepts an alternative overseer generator, to adjust behavior
 /// for integration tests as needed.
@@ -415,6 +417,13 @@ pub fn run() -> Result<()> {
 			#[cfg(not(feature = "polkadot-native"))]
 			panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
 		},
+		Some(Subcommand::HostPerfCheck) => {
+			let mut builder = sc_cli::LoggerBuilder::new("");
+			builder.with_colors(true);
+			builder.init()?;
+
+			host_perf_check()
+		},
 		Some(Subcommand::Key(cmd)) => Ok(cmd.run(&cli)?),
 		#[cfg(feature = "try-runtime")]
 		Some(Subcommand::TryRuntime(cmd)) => {

polkadot/cli/src/error.rs

@@ -0,0 +1,36 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+	#[error(transparent)]
+	PolkadotService(#[from] service::Error),
+
+	#[error(transparent)]
+	SubstrateCli(#[from] sc_cli::Error),
+
+	#[error(transparent)]
+	SubstrateService(#[from] sc_service::Error),
+
+	#[error(transparent)]
+	SubstrateTracing(#[from] sc_tracing::logging::Error),
+
+	#[error(transparent)]
+	PerfCheck(#[from] polkadot_performance_test::PerfCheckError),
+
+	#[error("Other: {0}")]
+	Other(String),
+}

polkadot/cli/src/host_perf_check.rs

@@ -0,0 +1,67 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use log::info;
+use polkadot_node_core_pvf::sp_maybe_compressed_blob;
+use polkadot_performance_test::{
+	measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS,
+	ERASURE_CODING_TIME_LIMIT, PVF_PREPARE_TIME_LIMIT, VALIDATION_CODE_BOMB_LIMIT,
+};
+use std::time::Duration;
+
+pub fn host_perf_check() -> Result<(), PerfCheckError> {
+	let wasm_code =
+		polkadot_performance_test::WASM_BINARY.ok_or(PerfCheckError::WasmBinaryMissing)?;
+
+	// Decompress the code before running checks.
+	let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT)
+		.or(Err(PerfCheckError::CodeDecompressionFailed))?;
+
+	info!("Running the performance checks...");
+
+	perf_check("PVF-prepare", PVF_PREPARE_TIME_LIMIT, || measure_pvf_prepare(code.as_ref()))?;
+
+	perf_check("Erasure-coding", ERASURE_CODING_TIME_LIMIT, || {
+		measure_erasure_coding(ERASURE_CODING_N_VALIDATORS, code.as_ref())
+	})?;
+
+	Ok(())
+}
+
+fn green_threshold(duration: Duration) -> Duration {
+	duration * 4 / 5
+}
+
+fn perf_check(
+	test_name: &str,
+	time_limit: Duration,
+	test: impl Fn() -> Result<Duration, PerfCheckError>,
+) -> Result<(), PerfCheckError> {
+	let elapsed = test()?;
+
+	if elapsed < green_threshold(time_limit) {
+		info!("🟢 {} performance check passed, elapsed: {:?}", test_name, elapsed);
+		Ok(())
+	} else if elapsed <= time_limit {
+		info!(
+			"🟡 {} performance check passed, {:?} limit almost exceeded, elapsed: {:?}",
+			test_name, time_limit, elapsed
+		);
+		Ok(())
+	} else {
+		Err(PerfCheckError::TimeOut { elapsed, limit: time_limit })
+	}
+}

polkadot/cli/src/lib.rs

@@ -22,6 +22,10 @@
 mod cli;
 #[cfg(feature = "cli")]
 mod command;
+#[cfg(feature = "cli")]
+mod error;
+#[cfg(all(feature = "cli", build_type = "release"))]
+mod host_perf_check;
 
 #[cfg(feature = "full-node")]
 pub use service::RuntimeApiCollection;

polkadot/erasure-coding/src/lib.rs

@@ -135,7 +135,7 @@ pub fn obtain_chunks_v1(n_validators: usize, data: &AvailableData) -> Result<Vec
 /// Obtain erasure-coded chunks, one for each validator.
 ///
 /// Works only up to 65536 validators, and `n_validators` must be non-zero.
-fn obtain_chunks<T: Encode>(n_validators: usize, data: &T) -> Result<Vec<Vec<u8>>, Error> {
+pub fn obtain_chunks<T: Encode>(n_validators: usize, data: &T) -> Result<Vec<Vec<u8>>, Error> {
 	let params = code_params(n_validators)?;
 	let encoded = data.encode();
 
@@ -186,7 +186,7 @@ where
 /// are provided, recovery is not possible.
 ///
 /// Works only up to 65536 validators, and `n_validators` must be non-zero.
-fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
+pub fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
 where
 	I: IntoIterator<Item = (&'a [u8], usize)>,
 {

polkadot/node/core/pvf/src/lib.rs

@@ -102,4 +102,9 @@ pub use metrics::Metrics;
 pub use execute::worker_entrypoint as execute_worker_entrypoint;
 pub use prepare::worker_entrypoint as prepare_worker_entrypoint;
 
+pub use executor_intf::{prepare, prevalidate};
+
+pub use sc_executor_common;
+pub use sp_maybe_compressed_blob;
+
 const LOG_TARGET: &str = "parachain::pvf";

polkadot/node/test/performance-test/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "polkadot-performance-test"
+version = "0.9.13"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+thiserror = "1.0.30"
+quote = "1.0.10"
+env_logger = "0.9"
+log = "0.4"
+
+polkadot-node-core-pvf = { path = "../../core/pvf" }
+polkadot-erasure-coding = { path = "../../../erasure-coding" }
+polkadot-node-primitives = { path = "../../primitives" }
+
+kusama-runtime = { path = "../../../runtime/kusama" }
+
+[[bin]]
+name = "gen-ref-constants"
+path = "src/gen_ref_constants.rs"

polkadot/node/test/performance-test/build.rs

@@ -0,0 +1,21 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+fn main() {
+	if let Ok(profile) = std::env::var("PROFILE") {
+		println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
+	}
+}

polkadot/node/test/performance-test/src/constants.rs

@@ -0,0 +1,22 @@
+// Copyright 2017-2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This file was automatically generated by `gen-ref-constants`.
+//! Do not edit manually!
+
+use std::time::Duration;
+pub const PVF_PREPARE_TIME_LIMIT: Duration = Duration::from_millis(4910u64);
+pub const ERASURE_CODING_TIME_LIMIT: Duration = Duration::from_millis(466u64);

polkadot/node/test/performance-test/src/gen_ref_constants.rs

@@ -0,0 +1,100 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generate reference performance check results.
+
+use polkadot_performance_test::PerfCheckError;
+
+fn main() -> Result<(), PerfCheckError> {
+	#[cfg(build_type = "release")]
+	{
+		run::run()
+	}
+	#[cfg(not(build_type = "release"))]
+	{
+		Err(PerfCheckError::WrongBuildType)
+	}
+}
+
+#[cfg(build_type = "release")]
+mod run {
+	use polkadot_node_core_pvf::sp_maybe_compressed_blob;
+	use polkadot_node_primitives::VALIDATION_CODE_BOMB_LIMIT;
+	use polkadot_performance_test::{
+		measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS,
+	};
+	use std::{
+		fs::OpenOptions,
+		io::{self, Write},
+		time::Duration,
+	};
+
+	const WARM_UP_RUNS: usize = 16;
+	const FILE_HEADER: &str = include_str!("../../../../file_header.txt");
+	const DOC_COMMENT: &str = "//! This file was automatically generated by `gen-ref-constants`.\n//! Do not edit manually!";
+	const FILE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/constants.rs");
+
+	fn save_constants(pvf_prepare: Duration, erasure_coding: Duration) -> io::Result<()> {
+		let mut output =
+			OpenOptions::new().truncate(true).create(true).write(true).open(FILE_PATH)?;
+
+		writeln!(output, "{}\n\n{}\n", FILE_HEADER, DOC_COMMENT)?;
+
+		let pvf_prepare_millis = pvf_prepare.as_millis() as u64;
+		let erasure_coding_millis = erasure_coding.as_millis() as u64;
+
+		let token_stream = quote::quote! {
+			use std::time::Duration;
+
+			pub const PVF_PREPARE_TIME_LIMIT: Duration = Duration::from_millis(#pvf_prepare_millis);
+			pub const ERASURE_CODING_TIME_LIMIT: Duration = Duration::from_millis(#erasure_coding_millis);
+		};
+
+		writeln!(output, "{}", token_stream.to_string())?;
+		Ok(())
+	}
+
+	pub fn run() -> Result<(), PerfCheckError> {
+		let _ = env_logger::builder().filter(None, log::LevelFilter::Info).try_init();
+
+		let wasm_code =
+			polkadot_performance_test::WASM_BINARY.ok_or(PerfCheckError::WasmBinaryMissing)?;
+
+		log::info!("Running the benchmark, number of iterations: {}", WARM_UP_RUNS);
+
+		let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT)
+			.or(Err(PerfCheckError::CodeDecompressionFailed))?;
+
+		let (pvf_prepare_time, erasure_coding_time) = (1..=WARM_UP_RUNS)
+			.map(|i| {
+				if i - 1 > 0 && (i - 1) % 5 == 0 {
+					log::info!("{} iterations done", i - 1);
+				}
+				(
+					measure_pvf_prepare(code.as_ref()),
+					measure_erasure_coding(ERASURE_CODING_N_VALIDATORS, code.as_ref()),
+				)
+			})
+			.last()
+			.expect("`WARM_UP_RUNS` is greater than 1 and thus we have at least one element; qed");
+
+		save_constants(pvf_prepare_time?, erasure_coding_time?)?;
+
+		log::info!("Successfully stored new reference values at {:?}. Make sure to format the file via `cargo +nightly fmt`", FILE_PATH);
+
+		Ok(())
+	}
+}

polkadot/node/test/performance-test/src/lib.rs

@@ -0,0 +1,87 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A Polkadot performance tests utilities.
+
+use polkadot_erasure_coding::{obtain_chunks, reconstruct};
+use polkadot_node_core_pvf::{sc_executor_common, sp_maybe_compressed_blob};
+use std::time::{Duration, Instant};
+
+mod constants;
+
+pub use constants::*;
+pub use polkadot_node_primitives::VALIDATION_CODE_BOMB_LIMIT;
+
+/// Value used for reference benchmark of erasure-coding.
+pub const ERASURE_CODING_N_VALIDATORS: usize = 1024;
+
+pub use kusama_runtime::WASM_BINARY;
+
+#[allow(missing_docs)]
+#[derive(thiserror::Error, Debug)]
+pub enum PerfCheckError {
+	#[error("This subcommand is only available in release mode")]
+	WrongBuildType,
+
+	#[error("No wasm code found for running the performance test")]
+	WasmBinaryMissing,
+
+	#[error("Failed to decompress wasm code")]
+	CodeDecompressionFailed,
+
+	#[error(transparent)]
+	Wasm(#[from] sc_executor_common::error::WasmError),
+
+	#[error(transparent)]
+	ErasureCoding(#[from] polkadot_erasure_coding::Error),
+
+	#[error(transparent)]
+	Io(#[from] std::io::Error),
+
+	#[error(
+		"Performance check not passed: exceeded the {limit:?} time limit, elapsed: {elapsed:?}"
+	)]
+	TimeOut { elapsed: Duration, limit: Duration },
+}
+
+/// Measures the time it takes to compile arbitrary wasm code.
+pub fn measure_pvf_prepare(wasm_code: &[u8]) -> Result<Duration, PerfCheckError> {
+	let start = Instant::now();
+
+	let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT)
+		.or(Err(PerfCheckError::CodeDecompressionFailed))?;
+
+	// Recreate the pipeline from the pvf prepare worker.
+	let blob = polkadot_node_core_pvf::prevalidate(code.as_ref()).map_err(PerfCheckError::from)?;
+	polkadot_node_core_pvf::prepare(blob).map_err(PerfCheckError::from)?;
+
+	Ok(start.elapsed())
+}
+
+/// Measure the time it takes to break arbitrary data into chunks and reconstruct it back.
+pub fn measure_erasure_coding(
+	n_validators: usize,
+	data: &[u8],
+) -> Result<Duration, PerfCheckError> {
+	let start = Instant::now();
+
+	let chunks = obtain_chunks(n_validators, &data)?;
+	let indexed_chunks = chunks.iter().enumerate().map(|(i, chunk)| (chunk.as_slice(), i));
+
+	let _: Vec<u8> = reconstruct(n_validators, indexed_chunks)?;
+
+	Ok(start.elapsed())
+}