Co #11164: Sub-commands for benchmark (#1156)

* para-template: Add bench commands

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* collator: Add bench commands

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Test benchmark commands

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove comments

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* update lockfile for {"polkadot"}

* Remove benchmark block test as the collator cannot produce blocks on its own

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: parity-processbot <>

cumulus/parachain-template/node/src/cli.rs

@@ -34,8 +34,9 @@ pub enum Subcommand {
 	/// Revert the chain to a previous state.
 	Revert(sc_cli::RevertCmd),
 
-	/// The custom benchmark subcommmand benchmarking runtime pallets.
-	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
+	/// Sub-commands concerned with benchmarking.
+	/// The pallet benchmarking moved to the `pallet` sub-command.
+	#[clap(subcommand)]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
 	/// Try some testing command against a specified runtime state.

cumulus/parachain-template/node/src/command.rs

@@ -6,6 +6,7 @@ use crate::{
 use codec::Encode;
 use cumulus_client_service::genesis::generate_genesis_block;
 use cumulus_primitives_core::ParaId;
+use frame_benchmarking_cli::BenchmarkCmd;
 use log::info;
 use parachain_template_runtime::{Block, RuntimeApi};
 use polkadot_parachain::primitives::AccountIdConversion;
@@ -231,16 +232,38 @@ pub fn run() -> Result<()> {
 
 			Ok(())
 		},
-		Some(Subcommand::Benchmark(cmd)) =>
-			if cfg!(feature = "runtime-benchmarks") {
-				let runner = cli.create_runner(cmd)?;
+		Some(Subcommand::Benchmark(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			// Switch on the concrete benchmark sub-command-
+			match cmd {
+				BenchmarkCmd::Pallet(cmd) =>
+					if cfg!(feature = "runtime-benchmarks") {
+						runner.sync_run(|config| cmd.run::<Block, TemplateRuntimeExecutor>(config))
+					} else {
+						Err("Benchmarking wasn't enabled when building the node. \
+					You can enable it with `--features runtime-benchmarks`."
+							.into())
+					},
+				BenchmarkCmd::Block(cmd) => runner.sync_run(|config| {
+					let partials = new_partial::<RuntimeApi, TemplateRuntimeExecutor, _>(
+						&config,
+						crate::service::parachain_build_import_queue,
+					)?;
+					cmd.run(partials.client)
+				}),
+				BenchmarkCmd::Storage(cmd) => runner.sync_run(|config| {
+					let partials = new_partial::<RuntimeApi, TemplateRuntimeExecutor, _>(
+						&config,
+						crate::service::parachain_build_import_queue,
+					)?;
+					let db = partials.backend.expose_db();
+					let storage = partials.backend.expose_storage();
 
-				runner.sync_run(|config| cmd.run::<Block, TemplateRuntimeExecutor>(config))
-			} else {
-				Err("Benchmarking wasn't enabled when building the node. \
-				You can enable it with `--features runtime-benchmarks`."
-					.into())
-			},
+					cmd.run(config, partials.client.clone(), db, storage)
+				}),
+				BenchmarkCmd::Overhead(_) => Err("Unsupported benchmarking command".into()),
+			}
+		},
 		Some(Subcommand::TryRuntime(cmd)) => {
 			if cfg!(feature = "try-runtime") {
 				let runner = cli.create_runner(cmd)?;

cumulus/polkadot-parachains/src/cli.rs

@@ -51,8 +51,9 @@ pub enum Subcommand {
 	/// Revert the chain to a previous state.
 	Revert(sc_cli::RevertCmd),
 
-	/// The custom benchmark subcommmand benchmarking runtime pallets.
-	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
+	/// Sub-commands concerned with benchmarking.
+	/// The pallet benchmarking moved to the `pallet` sub-command.
+	#[clap(subcommand)]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
 	/// Try some testing command against a specified runtime state.

cumulus/polkadot-parachains/src/command.rs

@@ -25,6 +25,7 @@ use crate::{
 use codec::Encode;
 use cumulus_client_service::genesis::generate_genesis_block;
 use cumulus_primitives_core::ParaId;
+use frame_benchmarking_cli::BenchmarkCmd;
 use log::info;
 use parachains_common::{AuraId, StatemintAuraId};
 use polkadot_parachain::primitives::AccountIdConversion;
@@ -268,6 +269,33 @@ fn extract_genesis_wasm(chain_spec: &Box<dyn sc_service::ChainSpec>) -> Result<V
 		.ok_or_else(|| "Could not find wasm file in genesis state!".into())
 }
 
+/// Creates partial components for the runtimes that are supported by the benchmarks.
+macro_rules! construct_benchmark_partials {
+	($config:expr, |$partials:ident| $code:expr) => {
+		if $config.chain_spec.is_statemine() {
+			let $partials = new_partial::<statemine_runtime::RuntimeApi, _>(
+				&$config,
+				crate::service::statemint_build_import_queue::<_, AuraId>,
+			)?;
+			$code
+		} else if $config.chain_spec.is_westmint() {
+			let $partials = new_partial::<westmint_runtime::RuntimeApi, _>(
+				&$config,
+				crate::service::statemint_build_import_queue::<_, AuraId>,
+			)?;
+			$code
+		} else if $config.chain_spec.is_statemint() {
+			let $partials = new_partial::<statemint_runtime::RuntimeApi, _>(
+				&$config,
+				crate::service::statemint_build_import_queue::<_, StatemintAuraId>,
+			)?;
+			$code
+		} else {
+			Err("The chain is not supported".into())
+		}
+	};
+}
+
 macro_rules! construct_async_run {
 	(|$components:ident, $cli:ident, $cmd:ident, $config:ident| $( $code:tt )* ) => {{
 		let runner = $cli.create_runner($cmd)?;
@@ -439,23 +467,43 @@ pub fn run() -> Result<()> {
 
 			Ok(())
 		},
-		Some(Subcommand::Benchmark(cmd)) =>
-			if cfg!(feature = "runtime-benchmarks") {
-				let runner = cli.create_runner(cmd)?;
-				if runner.config().chain_spec.is_statemine() {
-					runner.sync_run(|config| cmd.run::<Block, StatemineRuntimeExecutor>(config))
-				} else if runner.config().chain_spec.is_westmint() {
-					runner.sync_run(|config| cmd.run::<Block, WestmintRuntimeExecutor>(config))
-				} else if runner.config().chain_spec.is_statemint() {
-					runner.sync_run(|config| cmd.run::<Block, StatemintRuntimeExecutor>(config))
-				} else {
-					Err("Chain doesn't support benchmarking".into())
-				}
-			} else {
-				Err("Benchmarking wasn't enabled when building the node. \
+		Some(Subcommand::Benchmark(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+
+			// Switch on the concrete benchmark sub-command-
+			match cmd {
+				BenchmarkCmd::Pallet(cmd) =>
+					if cfg!(feature = "runtime-benchmarks") {
+						runner.sync_run(|config| {
+							if config.chain_spec.is_statemine() {
+								cmd.run::<Block, StatemineRuntimeExecutor>(config)
+							} else if config.chain_spec.is_westmint() {
+								cmd.run::<Block, WestmintRuntimeExecutor>(config)
+							} else if config.chain_spec.is_statemint() {
+								cmd.run::<Block, StatemintRuntimeExecutor>(config)
+							} else {
+								Err("Chain doesn't support benchmarking".into())
+							}
+						})
+					} else {
+						Err("Benchmarking wasn't enabled when building the node. \
 				You can enable it with `--features runtime-benchmarks`."
-					.into())
-			},
+							.into())
+					},
+				BenchmarkCmd::Block(cmd) => runner.sync_run(|config| {
+					construct_benchmark_partials!(config, |partials| cmd.run(partials.client))
+				}),
+				BenchmarkCmd::Storage(cmd) => runner.sync_run(|config| {
+					construct_benchmark_partials!(config, |partials| {
+						let db = partials.backend.expose_db();
+						let storage = partials.backend.expose_storage();
+
+						cmd.run(config, partials.client.clone(), db, storage)
+					})
+				}),
+				BenchmarkCmd::Overhead(_) => Err("Unsupported benchmarking command".into()),
+			}
+		},
 		Some(Subcommand::TryRuntime(cmd)) => {
 			if cfg!(feature = "try-runtime") {
 				// grab the task manager.

cumulus/polkadot-parachains/tests/benchmark_storage_works.rs

@@ -0,0 +1,42 @@
+use assert_cmd::cargo::cargo_bin;
+use std::{
+	path::Path,
+	process::{Command, ExitStatus},
+};
+use tempfile::tempdir;
+
+/// The runtimes that this command supports.
+static RUNTIMES: [&'static str; 3] = ["westmint", "statemine", "statemint"];
+
+/// The `benchmark storage` command works for the dev runtimes.
+#[test]
+#[ignore]
+fn benchmark_storage_works() {
+	for runtime in RUNTIMES {
+		let tmp_dir = tempdir().expect("could not create a temp dir");
+		let base_path = tmp_dir.path();
+		let runtime = format!("{}-dev", runtime);
+
+		// Benchmarking the storage works and creates the weight file.
+		assert!(benchmark_storage("rocksdb", &runtime, base_path).success());
+		assert!(base_path.join("rocksdb_weights.rs").exists());
+
+		assert!(benchmark_storage("paritydb", &runtime, base_path).success());
+		assert!(base_path.join("paritydb_weights.rs").exists());
+	}
+}
+
+/// Invoke the `benchmark storage` sub-command for the given database and runtime.
+fn benchmark_storage(db: &str, runtime: &str, base_path: &Path) -> ExitStatus {
+	Command::new(cargo_bin("polkadot-collator"))
+		.args(&["benchmark", "storage", "--chain", runtime])
+		.arg("--db")
+		.arg(db)
+		.arg("--weight-path")
+		.arg(base_path)
+		.args(["--state-version", "0"])
+		.args(["--warmups", "0"])
+		.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
+		.status()
+		.unwrap()
+}