Fix benchmarking tests (#5791)

* Re-add tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Test with release profile

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Re-add tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Test with diener

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Revert "Test with release profile"

This reverts commit d5384c5b603e1f8b7d14871ee57d2661e61bdb75.

* Revert "Test with diener"

This reverts commit 9e035759c780c2dfaf431456c8d0ff1a78e68d64.

* cargo update -p sp-io

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update tests/benchmark_block.rs

Co-authored-by: Chevdor <chevdor@users.noreply.github.com>

Co-authored-by: Chevdor <chevdor@users.noreply.github.com>

polkadot/tests/benchmark_block.rs

@@ -0,0 +1,94 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+// Unix only since it uses signals.
+#![cfg(unix)]
+
+use assert_cmd::cargo::cargo_bin;
+use nix::{
+	sys::signal::{kill, Signal::SIGINT},
+	unistd::Pid,
+};
+use std::{
+	path::Path,
+	process::{self, Command},
+	result::Result,
+	time::Duration,
+};
+use tempfile::tempdir;
+
+pub mod common;
+
+static RUNTIMES: [&'static str; 4] = ["polkadot", "kusama", "westend", "rococo"];
+
+/// `benchmark block` works for all dev runtimes using the wasm executor.
+#[tokio::test]
+async fn benchmark_block_works() {
+	for runtime in RUNTIMES {
+		let tmp_dir = tempdir().expect("could not create a temp dir");
+		let base_path = tmp_dir.path();
+		let runtime = format!("{}-dev", runtime);
+
+		// Build a chain with a single block.
+		build_chain(&runtime, base_path).await.unwrap();
+		// Benchmark the one block.
+		benchmark_block(&runtime, base_path, 1).unwrap();
+	}
+}
+
+/// Builds a chain with one block for the given runtime and base path.
+async fn build_chain(runtime: &str, base_path: &Path) -> Result<(), String> {
+	let mut cmd = Command::new(cargo_bin("polkadot"))
+		.stdout(process::Stdio::piped())
+		.stderr(process::Stdio::piped())
+		.args(["--chain", &runtime, "--force-authoring", "--alice"])
+		.arg("-d")
+		.arg(base_path)
+		.arg("--no-hardware-benchmarks")
+		.spawn()
+		.unwrap();
+
+	let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap());
+
+	// Wait for the chain to produce one block.
+	let ok = common::wait_n_finalized_blocks(1, Duration::from_secs(60), &ws_url).await;
+	// Send SIGINT to node.
+	kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap();
+	// Wait for the node to handle it and exit.
+	assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default());
+
+	ok.map_err(|e| format!("Node did not build the chain: {:?}", e))
+}
+
+/// Benchmarks the given block with the wasm executor.
+fn benchmark_block(runtime: &str, base_path: &Path, block: u32) -> Result<(), String> {
+	// Invoke `benchmark block` with all options to make sure that they are valid.
+	let status = Command::new(cargo_bin("polkadot"))
+		.args(["benchmark", "block", "--chain", &runtime])
+		.arg("-d")
+		.arg(base_path)
+		.args(["--from", &block.to_string(), "--to", &block.to_string()])
+		.args(["--repeat", "1"])
+		.args(["--execution", "wasm", "--wasm-execution", "compiled"])
+		.status()
+		.map_err(|e| format!("command failed: {:?}", e))?;
+
+	if !status.success() {
+		return Err("Command failed".into())
+	}
+
+	Ok(())
+}

polkadot/tests/benchmark_extrinsic.rs

@@ -0,0 +1,58 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::{process::Command, result::Result};
+
+static RUNTIMES: [&'static str; 4] = ["polkadot", "kusama", "westend", "rococo"];
+
+static EXTRINSICS: [(&'static str, &'static str); 2] =
+	[("system", "remark"), ("balances", "transfer_keep_alive")];
+
+/// `benchmark extrinsic` works for all dev runtimes and some extrinsics.
+#[test]
+fn benchmark_extrinsic_works() {
+	for runtime in RUNTIMES {
+		for (pallet, extrinsic) in EXTRINSICS {
+			let runtime = format!("{}-dev", runtime);
+			assert!(benchmark_extrinsic(&runtime, pallet, extrinsic).is_ok());
+		}
+	}
+}
+
+/// `benchmark extrinsic` rejects all non-dev runtimes.
+#[test]
+fn benchmark_extrinsic_rejects_non_dev_runtimes() {
+	for runtime in RUNTIMES {
+		assert!(benchmark_extrinsic(runtime, "system", "remark").is_err());
+	}
+}
+
+fn benchmark_extrinsic(runtime: &str, pallet: &str, extrinsic: &str) -> Result<(), String> {
+	let status = Command::new(cargo_bin("polkadot"))
+		.args(["benchmark", "extrinsic", "--chain", &runtime])
+		.args(&["--pallet", pallet, "--extrinsic", extrinsic])
+		// Run with low repeats for faster execution.
+		.args(["--repeat=1", "--warmup=1", "--max-ext-per-block=1"])
+		.status()
+		.map_err(|e| format!("command failed: {:?}", e))?;
+
+	if !status.success() {
+		return Err("Command failed".into())
+	}
+
+	Ok(())
+}

polkadot/tests/benchmark_overhead.rs

@@ -0,0 +1,67 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::{process::Command, result::Result};
+use tempfile::tempdir;
+
+static RUNTIMES: [&'static str; 4] = ["polkadot", "kusama", "westend", "rococo"];
+
+/// `benchmark overhead` works for all dev runtimes.
+#[test]
+fn benchmark_overhead_works() {
+	for runtime in RUNTIMES {
+		let runtime = format!("{}-dev", runtime);
+		assert!(benchmark_overhead(runtime).is_ok());
+	}
+}
+
+/// `benchmark overhead` rejects all non-dev runtimes.
+#[test]
+fn benchmark_overhead_rejects_non_dev_runtimes() {
+	for runtime in RUNTIMES {
+		assert!(benchmark_overhead(runtime.into()).is_err());
+	}
+}
+
+fn benchmark_overhead(runtime: String) -> Result<(), String> {
+	let tmp_dir = tempdir().expect("could not create a temp dir");
+	let base_path = tmp_dir.path();
+
+	// Invoke `benchmark overhead` with all options to make sure that they are valid.
+	let status = Command::new(cargo_bin("polkadot"))
+		.args(["benchmark", "overhead", "--chain", &runtime])
+		.arg("-d")
+		.arg(base_path)
+		.arg("--weight-path")
+		.arg(base_path)
+		.args(["--warmup", "5", "--repeat", "5"])
+		.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
+		// Only put 5 extrinsics into the block otherwise it takes forever to build it
+		// especially for a non-release builds.
+		.args(["--max-ext-per-block", "5"])
+		.status()
+		.map_err(|e| format!("command failed: {:?}", e))?;
+
+	if !status.success() {
+		return Err("Command failed".into())
+	}
+
+	// Weight files have been created.
+	assert!(base_path.join("block_weights.rs").exists());
+	assert!(base_path.join("extrinsic_weights.rs").exists());
+	Ok(())
+}

polkadot/tests/common.rs

@@ -91,6 +91,6 @@ pub fn find_ws_url_from_output(read: impl Read + Send) -> (String, String) {
 
 			Some(format!("ws://{}", sock_addr))
 		})
-		.expect("We should get a WebSocket address");
+		.expect(&format!("Could not find WebSocket address in process output:\n{}", &data));
 	(ws_url, data)
 }