Sub-commands for benchmark (#11164)

* Restructure benchmark commands

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add benchmark block test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fixup imports

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Extend error message

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Apply suggestions from code review

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add commands to node-template

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

substrate/Cargo.lock

@@ -5089,8 +5089,10 @@ dependencies = [
  "clap 3.1.6",
  "frame-benchmarking",
  "frame-benchmarking-cli",
+ "frame-system",
  "jsonrpc-core",
  "node-template-runtime",
+ "pallet-transaction-payment",
  "pallet-transaction-payment-rpc",
  "sc-basic-authorship",
  "sc-cli",
@@ -5113,6 +5115,8 @@ dependencies = [
  "sp-consensus-aura",
  "sp-core",
  "sp-finality-grandpa",
+ "sp-inherents",
+ "sp-keyring",
  "sp-runtime",
  "sp-timestamp",
  "substrate-build-script-utils",

substrate/bin/node-template/node/Cargo.toml

@@ -36,6 +36,10 @@ sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/final
 sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
 sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" }
+sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" }
+sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" }
+frame-system = { version = "4.0.0-dev", path = "../../../frame/system" }
+pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" }
 
 # These dependencies are used for the node template's RPCs
 jsonrpc-core = "18.0.0"

substrate/bin/node-template/node/src/cli.rs

@@ -36,8 +36,8 @@ pub enum Subcommand {
 	/// Revert the chain to a previous state.
 	Revert(sc_cli::RevertCmd),
 
-	/// The custom benchmark subcommand benchmarking runtime pallets.
-	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
+	/// Sub-commands concerned with benchmarking.
+	#[clap(subcommand)]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
 	/// Try some command against runtime state.

substrate/bin/node-template/node/src/command.rs

@@ -1,11 +1,14 @@
 use crate::{
 	chain_spec,
 	cli::{Cli, Subcommand},
+	command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder},
 	service,
 };
+use frame_benchmarking_cli::BenchmarkCmd;
 use node_template_runtime::Block;
 use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
+use std::sync::Arc;
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -102,16 +105,41 @@ pub fn run() -> sc_cli::Result<()> {
 				Ok((cmd.run(client, backend, Some(aux_revert)), task_manager))
 			})
 		},
-		Some(Subcommand::Benchmark(cmd)) =>
-			if cfg!(feature = "runtime-benchmarks") {
-				let runner = cli.create_runner(cmd)?;
+		Some(Subcommand::Benchmark(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
 
-				runner.sync_run(|config| cmd.run::<Block, service::ExecutorDispatch>(config))
-			} else {
-				Err("Benchmarking wasn't enabled when building the node. You can enable it with \
-				     `--features runtime-benchmarks`."
-					.into())
-			},
+			runner.sync_run(|config| {
+				let PartialComponents { client, backend, .. } = service::new_partial(&config)?;
+
+				// This switch needs to be in the client, since the client decides
+				// which sub-commands it wants to support.
+				match cmd {
+					BenchmarkCmd::Pallet(cmd) => {
+						if !cfg!(feature = "runtime-benchmarks") {
+							return Err(
+								"Runtime benchmarking wasn't enabled when building the node. \
+							You can enable it with `--features runtime-benchmarks`."
+									.into(),
+							)
+						}
+
+						cmd.run::<Block, service::ExecutorDispatch>(config)
+					},
+					BenchmarkCmd::Block(cmd) => cmd.run(client),
+					BenchmarkCmd::Storage(cmd) => {
+						let db = backend.expose_db();
+						let storage = backend.expose_storage();
+
+						cmd.run(config, client, db, storage)
+					},
+					BenchmarkCmd::Overhead(cmd) => {
+						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+
+						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+					},
+				}
+			})
+		},
 		#[cfg(feature = "try-runtime")]
 		Some(Subcommand::TryRuntime(cmd)) => {
 			let runner = cli.create_runner(cmd)?;

substrate/bin/node-template/node/src/command_helper.rs

@@ -0,0 +1,131 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Contains code to setup the command invocations in [`super::command`] which would
+//! otherwise bloat that module.
+
+use crate::service::FullClient;
+
+use node_template_runtime as runtime;
+use runtime::SystemCall;
+use sc_cli::Result;
+use sc_client_api::BlockBackend;
+use sp_core::{Encode, Pair};
+use sp_inherents::{InherentData, InherentDataProvider};
+use sp_keyring::Sr25519Keyring;
+use sp_runtime::{OpaqueExtrinsic, SaturatedConversion};
+
+use std::{sync::Arc, time::Duration};
+
+/// Generates extrinsics for the `benchmark overhead` command.
+///
+/// Note: Should only be used for benchmarking.
+pub struct BenchmarkExtrinsicBuilder {
+	client: Arc<FullClient>,
+}
+
+impl BenchmarkExtrinsicBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>) -> Self {
+		Self { client }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
+	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic(
+			self.client.as_ref(),
+			acc,
+			SystemCall::remark { remark: vec![] }.into(),
+			nonce,
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
+/// Create a transaction using the given `call`.
+///
+/// Note: Should only be used for benchmarking.
+pub fn create_benchmark_extrinsic(
+	client: &FullClient,
+	sender: sp_core::sr25519::Pair,
+	call: runtime::Call,
+	nonce: u32,
+) -> runtime::UncheckedExtrinsic {
+	let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed");
+	let best_hash = client.chain_info().best_hash;
+	let best_block = client.chain_info().best_number;
+
+	let period = runtime::BlockHashCount::get()
+		.checked_next_power_of_two()
+		.map(|c| c / 2)
+		.unwrap_or(2) as u64;
+	let extra: runtime::SignedExtra = (
+		frame_system::CheckNonZeroSender::<runtime::Runtime>::new(),
+		frame_system::CheckSpecVersion::<runtime::Runtime>::new(),
+		frame_system::CheckTxVersion::<runtime::Runtime>::new(),
+		frame_system::CheckGenesis::<runtime::Runtime>::new(),
+		frame_system::CheckEra::<runtime::Runtime>::from(sp_runtime::generic::Era::mortal(
+			period,
+			best_block.saturated_into(),
+		)),
+		frame_system::CheckNonce::<runtime::Runtime>::from(nonce),
+		frame_system::CheckWeight::<runtime::Runtime>::new(),
+		pallet_transaction_payment::ChargeTransactionPayment::<runtime::Runtime>::from(0),
+	);
+
+	let raw_payload = runtime::SignedPayload::from_raw(
+		call.clone(),
+		extra.clone(),
+		(
+			(),
+			runtime::VERSION.spec_version,
+			runtime::VERSION.transaction_version,
+			genesis_hash,
+			best_hash,
+			(),
+			(),
+			(),
+		),
+	);
+	let signature = raw_payload.using_encoded(|e| sender.sign(e));
+
+	runtime::UncheckedExtrinsic::new_signed(
+		call.clone(),
+		sp_runtime::AccountId32::from(sender.public()).into(),
+		runtime::Signature::Sr25519(signature.clone()),
+		extra.clone(),
+	)
+}
+
+/// Generates inherent data for the `benchmark overhead` command.
+///
+/// Note: Should only be used for benchmarking.
+pub fn inherent_benchmark_data() -> Result<InherentData> {
+	let mut inherent_data = InherentData::new();
+	let d = Duration::from_millis(0);
+	let timestamp = sp_timestamp::InherentDataProvider::new(d.into());
+
+	timestamp
+		.provide_inherent_data(&mut inherent_data)
+		.map_err(|e| format!("creating inherent data: {:?}", e))?;
+	Ok(inherent_data)
+}

substrate/bin/node-template/node/src/main.rs

@@ -6,6 +6,7 @@ mod chain_spec;
 mod service;
 mod cli;
 mod command;
+mod command_helper;
 mod rpc;
 
 fn main() -> sc_cli::Result<()> {

substrate/bin/node-template/node/src/service.rs

@@ -31,7 +31,7 @@ impl sc_executor::NativeExecutionDispatch for ExecutorDispatch {
 	}
 }
 
-type FullClient =
+pub(crate) type FullClient =
 	sc_service::TFullClient<Block, RuntimeApi, NativeElseWasmExecutor<ExecutorDispatch>>;
 type FullBackend = sc_service::TFullBackend<Block>;
 type FullSelectChain = sc_consensus::LongestChain<FullBackend, Block>;

substrate/bin/node-template/runtime/src/lib.rs

@@ -33,6 +33,7 @@ pub use frame_support::{
 	},
 	StorageValue,
 };
+pub use frame_system::Call as SystemCall;
 pub use pallet_balances::Call as BalancesCall;
 pub use pallet_timestamp::Call as TimestampCall;
 use pallet_transaction_payment::CurrencyAdapter;
@@ -306,6 +307,8 @@ pub type SignedExtra = (
 );
 /// Unchecked extrinsic type as expected by this runtime.
 pub type UncheckedExtrinsic = generic::UncheckedExtrinsic<Address, Call, Signature, SignedExtra>;
+/// The payload being signed in transactions.
+pub type SignedPayload = generic::SignedPayload<Call, SignedExtra>;
 /// Executive: handles dispatch to the various modules.
 pub type Executive = frame_executive::Executive<
 	Runtime,

substrate/bin/node/cli/src/cli.rs

@@ -38,28 +38,11 @@ pub enum Subcommand {
 	)]
 	Inspect(node_inspect::cli::InspectCmd),
 
-	/// The custom benchmark subcommmand benchmarking runtime pallets.
-	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
+	/// Sub-commands concerned with benchmarking.
+	/// The pallet benchmarking moved to the `pallet` sub-command.
+	#[clap(subcommand)]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
-	/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
-	#[clap(
-		name = "benchmark-block",
-		about = "Benchmark the execution time of historic blocks and compare it to their consumed weight."
-	)]
-	BenchmarkBlock(frame_benchmarking_cli::BlockCmd),
-
-	/// Sub command for benchmarking the per-block and per-extrinsic execution overhead.
-	#[clap(
-		name = "benchmark-overhead",
-		about = "Benchmark the per-block and per-extrinsic execution overhead."
-	)]
-	BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd),
-
-	/// Sub command for benchmarking the storage speed.
-	#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
-	BenchmarkStorage(frame_benchmarking_cli::StorageCmd),
-
 	/// Try some command against runtime state.
 	#[cfg(feature = "try-runtime")]
 	TryRuntime(try_runtime_cli::TryRuntimeCmd),

substrate/bin/node/cli/src/command.rs

@@ -16,11 +16,13 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
+use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder};
 use crate::{
 	chain_spec, service,
 	service::{new_partial, FullClient},
 	Cli, Subcommand,
 };
+use frame_benchmarking_cli::*;
 use node_executor::ExecutorDispatch;
 use node_primitives::Block;
 use node_runtime::RuntimeApi;
@@ -92,45 +94,39 @@ pub fn run() -> Result<()> {
 
 			runner.sync_run(|config| cmd.run::<Block, RuntimeApi, ExecutorDispatch>(config))
 		},
-		Some(Subcommand::Benchmark(cmd)) =>
-			if cfg!(feature = "runtime-benchmarks") {
-				let runner = cli.create_runner(cmd)?;
-
-				runner.sync_run(|config| cmd.run::<Block, ExecutorDispatch>(config))
-			} else {
-				Err("Benchmarking wasn't enabled when building the node. \
-				You can enable it with `--features runtime-benchmarks`."
-					.into())
-			},
-		Some(Subcommand::BenchmarkBlock(cmd)) => {
+		Some(Subcommand::Benchmark(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
-			runner.async_run(|config| {
-				let PartialComponents { client, task_manager, .. } = new_partial(&config)?;
-				Ok((cmd.run(client), task_manager))
-			})
-		},
-		Some(Subcommand::BenchmarkOverhead(cmd)) => {
-			let runner = cli.create_runner(cmd)?;
-			runner.async_run(|mut config| {
-				use super::command_helper::{inherent_data, ExtrinsicBuilder};
-				// We don't use the authority role since that would start producing blocks
-				// in the background which would mess with our benchmark.
-				config.role = sc_service::Role::Full;
 
-				let PartialComponents { client, task_manager, .. } = new_partial(&config)?;
-				let ext_builder = ExtrinsicBuilder::new(client.clone());
+			runner.sync_run(|config| {
+				let PartialComponents { client, backend, .. } = new_partial(&config)?;
 
-				Ok((cmd.run(config, client, inherent_data()?, Arc::new(ext_builder)), task_manager))
-			})
-		},
-		Some(Subcommand::BenchmarkStorage(cmd)) => {
-			let runner = cli.create_runner(cmd)?;
-			runner.async_run(|config| {
-				let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?;
-				let db = backend.expose_db();
-				let storage = backend.expose_storage();
+				// This switch needs to be in the client, since the client decides
+				// which sub-commands it wants to support.
+				match cmd {
+					BenchmarkCmd::Pallet(cmd) => {
+						if !cfg!(feature = "runtime-benchmarks") {
+							return Err(
+								"Runtime benchmarking wasn't enabled when building the node. \
+							You can enable it with `--features runtime-benchmarks`."
+									.into(),
+							)
+						}
 
-				Ok((cmd.run(config, client, db, storage), task_manager))
+						cmd.run::<Block, ExecutorDispatch>(config)
+					},
+					BenchmarkCmd::Block(cmd) => cmd.run(client),
+					BenchmarkCmd::Storage(cmd) => {
+						let db = backend.expose_db();
+						let storage = backend.expose_storage();
+
+						cmd.run(config, client, db, storage)
+					},
+					BenchmarkCmd::Overhead(cmd) => {
+						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+
+						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+					},
+				}
 			})
 		},
 		Some(Subcommand::Key(cmd)) => cmd.run(&cli),

substrate/bin/node/cli/src/command_helper.rs

@@ -29,19 +29,19 @@ use sp_runtime::OpaqueExtrinsic;
 
 use std::{sync::Arc, time::Duration};
 
-/// Generates extrinsics for the `benchmark-overhead` command.
-pub struct ExtrinsicBuilder {
+/// Generates extrinsics for the `benchmark overhead` command.
+pub struct BenchmarkExtrinsicBuilder {
 	client: Arc<FullClient>,
 }
 
-impl ExtrinsicBuilder {
+impl BenchmarkExtrinsicBuilder {
 	/// Creates a new [`Self`] from the given client.
 	pub fn new(client: Arc<FullClient>) -> Self {
 		Self { client }
 	}
 }
 
-impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder {
+impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
 	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
 		let acc = Sr25519Keyring::Bob.pair();
 		let extrinsic: OpaqueExtrinsic = create_extrinsic(
@@ -56,8 +56,8 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder {
 	}
 }
 
-/// Generates inherent data for the `benchmark-overhead` command.
-pub fn inherent_data() -> Result<InherentData> {
+/// Generates inherent data for the `benchmark overhead` command.
+pub fn inherent_benchmark_data() -> Result<InherentData> {
 	let mut inherent_data = InherentData::new();
 	let d = Duration::from_millis(0);
 	let timestamp = sp_timestamp::InherentDataProvider::new(d.into());

substrate/bin/node/cli/tests/benchmark_block_works.rs

@@ -0,0 +1,48 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// Unix only since it uses signals from [`common::run_node_for_a_while`].
+#![cfg(unix)]
+
+use assert_cmd::cargo::cargo_bin;
+use std::process::Command;
+use tempfile::tempdir;
+
+pub mod common;
+
+/// `benchmark block` works for the dev runtime using the wasm executor.
+#[tokio::test]
+async fn benchmark_block_works() {
+	let base_dir = tempdir().expect("could not create a temp dir");
+
+	common::run_node_for_a_while(base_dir.path(), &["--dev"]).await;
+
+	// Invoke `benchmark block` with all options to make sure that they are valid.
+	let status = Command::new(cargo_bin("substrate"))
+		.args(["benchmark", "block", "--dev"])
+		.arg("-d")
+		.arg(base_dir.path())
+		.args(["--pruning", "archive"])
+		.args(["--from", "1", "--to", "1"])
+		.args(["--repeat", "1"])
+		.args(["--execution", "wasm", "--wasm-execution", "compiled"])
+		.status()
+		.unwrap();
+
+	assert!(status.success())
+}

substrate/bin/node/cli/tests/benchmark_overhead_works.rs

@@ -20,7 +20,7 @@ use assert_cmd::cargo::cargo_bin;
 use std::process::Command;
 use tempfile::tempdir;
 
-/// Tests that the `benchmark-overhead` command works for the substrate dev runtime.
+/// Tests that the `benchmark overhead` command works for the substrate dev runtime.
 #[test]
 fn benchmark_overhead_works() {
 	let tmp_dir = tempdir().expect("could not create a temp dir");
@@ -29,7 +29,7 @@ fn benchmark_overhead_works() {
 	// Only put 10 extrinsics into the block otherwise it takes forever to build it
 	// especially for a non-release build.
 	let status = Command::new(cargo_bin("substrate"))
-		.args(&["benchmark-overhead", "--dev", "-d"])
+		.args(&["benchmark", "overhead", "--dev", "-d"])
 		.arg(base_path)
 		.arg("--weight-path")
 		.arg(base_path)

substrate/bin/node/cli/tests/benchmark_storage_works.rs

@@ -23,7 +23,7 @@ use std::{
 };
 use tempfile::tempdir;
 
-/// Tests that the `benchmark-storage` command works for the dev runtime.
+/// Tests that the `benchmark storage` command works for the dev runtime.
 #[test]
 fn benchmark_storage_works() {
 	let tmp_dir = tempdir().expect("could not create a temp dir");
@@ -39,7 +39,7 @@ fn benchmark_storage_works() {
 
 fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus {
 	Command::new(cargo_bin("substrate"))
-		.args(&["benchmark-storage", "--dev"])
+		.args(&["benchmark", "storage", "--dev"])
 		.arg("--db")
 		.arg(db)
 		.arg("--weight-path")

substrate/utils/frame/benchmarking-cli/src/block/bench.rs

@@ -34,7 +34,7 @@ use serde::Serialize;
 use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
 use thousands::Separable;
 
-use crate::storage::record::{StatSelect, Stats};
+use crate::shared::{StatSelect, Stats};
 
 /// Log target for printing block weight info.
 const LOG_TARGET: &'static str = "benchmark::block::weight";

substrate/utils/frame/benchmarking-cli/src/block/cmd.rs

@@ -29,7 +29,7 @@ use std::{fmt::Debug, sync::Arc};
 
 use super::bench::{Benchmark, BenchmarkParams};
 
-/// Benchmark the execution time historic blocks.
+/// Benchmark the execution time of historic blocks.
 ///
 /// This can be used to verify that blocks do not use more weight than they consumed
 /// in their `WeightInfo`. Example:
@@ -73,7 +73,7 @@ impl BlockCmd {
 	/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
 	///
 	/// Output will be printed to console.
-	pub async fn run<Block, BA, C>(&self, client: Arc<C>) -> Result<()>
+	pub fn run<Block, BA, C>(&self, client: Arc<C>) -> Result<()>
 	where
 		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
 		BA: ClientBackend<Block>,

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -15,144 +15,85 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod block;
-mod command;
-pub mod overhead;
-mod post_processing;
-mod storage;
-mod writer;
+//! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.
 
-use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
-use std::{fmt::Debug, path::PathBuf};
+mod block;
+mod overhead;
+mod pallet;
+mod shared;
+mod storage;
 
 pub use block::BlockCmd;
 pub use overhead::{ExtrinsicBuilder, OverheadCmd};
+pub use pallet::PalletCmd;
 pub use storage::StorageCmd;
 
-// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
-// used like crate names with `_`
-fn parse_pallet_name(pallet: &str) -> String {
-	pallet.replace("-", "_")
+use sc_cli::{CliConfiguration, DatabaseParams, ImportParams, PruningParams, Result, SharedParams};
+
+/// The root `benchmarking` command.
+///
+/// Has no effect itself besides printing a help menu of the sub-commands.
+#[derive(Debug, clap::Subcommand)]
+pub enum BenchmarkCmd {
+	Pallet(PalletCmd),
+	Storage(StorageCmd),
+	Overhead(OverheadCmd),
+	Block(BlockCmd),
 }
 
-/// The `benchmark` command used to benchmark FRAME Pallets.
-#[derive(Debug, clap::Parser)]
-pub struct BenchmarkCmd {
-	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
-	#[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")]
-	pub pallet: Option<String>,
-
-	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
-	#[clap(short, long, required_unless_present = "list")]
-	pub extrinsic: Option<String>,
-
-	/// Select how many samples we should take across the variable components.
-	#[clap(short, long, default_value = "1")]
-	pub steps: u32,
-
-	/// Indicates lowest values for each of the component ranges.
-	#[clap(long = "low", use_value_delimiter = true)]
-	pub lowest_range_values: Vec<u32>,
-
-	/// Indicates highest values for each of the component ranges.
-	#[clap(long = "high", use_value_delimiter = true)]
-	pub highest_range_values: Vec<u32>,
-
-	/// Select how many repetitions of this benchmark should run from within the wasm.
-	#[clap(short, long, default_value = "1")]
-	pub repeat: u32,
-
-	/// Select how many repetitions of this benchmark should run from the client.
-	///
-	/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
-	#[clap(long, default_value = "1")]
-	pub external_repeat: u32,
-
-	/// Print the raw results in JSON format.
-	#[clap(long = "json")]
-	pub json_output: bool,
-
-	/// Write the raw results in JSON format into the given file.
-	#[clap(long, conflicts_with = "json-output")]
-	pub json_file: Option<PathBuf>,
-
-	/// Don't print the median-slopes linear regression analysis.
-	#[clap(long)]
-	pub no_median_slopes: bool,
-
-	/// Don't print the min-squares linear regression analysis.
-	#[clap(long)]
-	pub no_min_squares: bool,
-
-	/// Output the benchmarks to a Rust file at the given path.
-	#[clap(long)]
-	pub output: Option<PathBuf>,
-
-	/// Add a header file to your outputted benchmarks
-	#[clap(long)]
-	pub header: Option<PathBuf>,
-
-	/// Path to Handlebars template file used for outputting benchmark results. (Optional)
-	#[clap(long)]
-	pub template: Option<PathBuf>,
-
-	/// Which analysis function to use when outputting benchmarks:
-	/// * min-squares (default)
-	/// * median-slopes
-	/// * max (max of min squares and median slopes for each value)
-	#[clap(long)]
-	pub output_analysis: Option<String>,
-
-	/// Set the heap pages while running benchmarks. If not set, the default value from the client
-	/// is used.
-	#[clap(long)]
-	pub heap_pages: Option<u64>,
-
-	/// Disable verification logic when running benchmarks.
-	#[clap(long)]
-	pub no_verify: bool,
-
-	/// Display and run extra benchmarks that would otherwise not be needed for weight
-	/// construction.
-	#[clap(long)]
-	pub extra: bool,
-
-	/// Estimate PoV size.
-	#[clap(long)]
-	pub record_proof: bool,
-
-	#[allow(missing_docs)]
-	#[clap(flatten)]
-	pub shared_params: sc_cli::SharedParams,
-
-	/// The execution strategy that should be used for benchmarks
-	#[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)]
-	pub execution: Option<ExecutionStrategy>,
-
-	/// Method for executing Wasm runtime code.
-	#[clap(
-		long = "wasm-execution",
-		value_name = "METHOD",
-		possible_values = WasmExecutionMethod::variants(),
-		ignore_case = true,
-		default_value = DEFAULT_WASM_EXECUTION_METHOD,
-	)]
-	pub wasm_method: WasmExecutionMethod,
-
-	/// Limit the memory the database cache can use.
-	#[clap(long = "db-cache", value_name = "MiB", default_value = "1024")]
-	pub database_cache_size: u32,
-
-	/// List the benchmarks that match your query rather than running them.
-	///
-	/// When nothing is provided, we list all benchmarks.
-	#[clap(long)]
-	pub list: bool,
-
-	/// If enabled, the storage info is not displayed in the output next to the analysis.
-	///
-	/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
-	/// template for that purpose.
-	#[clap(long)]
-	pub no_storage_info: bool,
+/// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
+macro_rules! unwrap_cmd {
+	{
+		$self:expr,
+		$cmd:ident,
+		$code:expr
+	} => {
+		match $self {
+			BenchmarkCmd::Pallet($cmd) => $code,
+			BenchmarkCmd::Storage($cmd) => $code,
+			BenchmarkCmd::Overhead($cmd) => $code,
+			BenchmarkCmd::Block($cmd) => $code,
+		}
+	}
+}
+
+/// Forward the [`CliConfiguration`] trait implementation.
+///
+/// Each time a sub-command exposes a new config option, it must be added here.
+impl CliConfiguration for BenchmarkCmd {
+	fn shared_params(&self) -> &SharedParams {
+		unwrap_cmd! {
+			self, cmd, cmd.shared_params()
+		}
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.import_params()
+		}
+	}
+
+	fn database_params(&self) -> Option<&DatabaseParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.database_params()
+		}
+	}
+
+	fn pruning_params(&self) -> Option<&PruningParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.pruning_params()
+		}
+	}
+
+	fn state_cache_size(&self) -> Result<usize> {
+		unwrap_cmd! {
+			self, cmd, cmd.state_cache_size()
+		}
+	}
+
+	fn chain_id(&self, is_dev: bool) -> Result<String> {
+		unwrap_cmd! {
+			self, cmd, cmd.chain_id(is_dev)
+		}
+	}
 }

substrate/utils/frame/benchmarking-cli/src/overhead/bench.rs

@@ -36,7 +36,8 @@ use log::info;
 use serde::Serialize;
 use std::{marker::PhantomData, sync::Arc, time::Instant};
 
-use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats};
+use super::cmd::ExtrinsicBuilder;
+use crate::shared::Stats;
 
 /// Parameters to configure an *overhead* benchmark.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]

substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs

@@ -35,10 +35,10 @@ use crate::{
 		bench::{Benchmark, BenchmarkParams, BenchmarkType},
 		template::TemplateData,
 	},
-	post_processing::WeightParams,
+	shared::WeightParams,
 };
 
-/// Benchmarks the per-block and per-extrinsic execution overhead.
+/// Benchmark the execution overhead per-block and per-extrinsic.
 #[derive(Debug, Parser)]
 pub struct OverheadCmd {
 	#[allow(missing_docs)]
@@ -76,11 +76,11 @@ pub trait ExtrinsicBuilder {
 }
 
 impl OverheadCmd {
-	/// Measures the per-block and per-extrinsic execution overhead.
+	/// Measure the per-block and per-extrinsic execution overhead.
 	///
 	/// Writes the results to console and into two instances of the
 	/// `weights.hbs` template, one for each benchmark.
-	pub async fn run<Block, BA, C>(
+	pub fn run<Block, BA, C>(
 		&self,
 		cfg: Configuration,
 		client: Arc<C>,

substrate/utils/frame/benchmarking-cli/src/overhead/template.rs

@@ -28,7 +28,7 @@ use std::{env, fs, path::PathBuf};
 
 use crate::{
 	overhead::{bench::BenchmarkType, cmd::OverheadParams},
-	storage::record::Stats,
+	shared::{Stats, UnderscoreHelper},
 };
 
 static VERSION: &'static str = env!("CARGO_PKG_VERSION");
@@ -85,7 +85,7 @@ impl TemplateData {
 	pub fn write(&self, path: &Option<PathBuf>) -> Result<()> {
 		let mut handlebars = Handlebars::new();
 		// Format large integers with underscores.
-		handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper));
+		handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
 		// Don't HTML escape any characters.
 		handlebars.register_escape_fn(|s| -> String { s.to_string() });
 

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::BenchmarkCmd;
+use super::{writer, PalletCmd};
 use codec::{Decode, Encode};
 use frame_benchmarking::{
 	Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter,
@@ -87,7 +87,13 @@ fn combine_batches(
 		.collect::<Vec<_>>()
 }
 
-impl BenchmarkCmd {
+/// Explains possible reasons why the metadata for the benchmarking could not be found.
+const ERROR_METADATA_NOT_FOUND: &'static str = "Did not find the benchmarking metadata. \
+This could mean that you either did not build the node correctly with the \
+`--features runtime-benchmarks` flag, or the chain spec that you are using was \
+not created by a node that was compiled with the flag";
+
+impl PalletCmd {
 	/// Runs the command and benchmarks the chain.
 	pub fn run<BB, ExecDispatch>(&self, config: Configuration) -> Result<()>
 	where
@@ -165,7 +171,7 @@ impl BenchmarkCmd {
 			sp_core::testing::TaskExecutor::new(),
 		)
 		.execute(strategy.into())
-		.map_err(|e| format!("Error getting benchmark list: {}", e))?;
+		.map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?;
 
 		let (list, storage_info) =
 			<(Vec<BenchmarkList>, Vec<StorageInfo>) as Decode>::decode(&mut &result[..])
@@ -359,7 +365,7 @@ impl BenchmarkCmd {
 
 		// Create the weights.rs file.
 		if let Some(output_path) = &self.output {
-			crate::writer::write_results(&batches, &storage_info, output_path, self)?;
+			writer::write_results(&batches, &storage_info, output_path, self)?;
 		}
 
 		// Jsonify the result and write it to a file or stdout if desired.
@@ -414,11 +420,7 @@ impl BenchmarkCmd {
 
 			if !self.no_storage_info {
 				let mut comments: Vec<String> = Default::default();
-				crate::writer::add_storage_comments(
-					&mut comments,
-					&batch.db_results,
-					&storage_info,
-				);
+				writer::add_storage_comments(&mut comments, &batch.db_results, &storage_info);
 				println!("Raw Storage Info\n========");
 				for comment in comments {
 					println!("{}", comment);
@@ -469,7 +471,7 @@ impl BenchmarkCmd {
 	}
 }
 
-impl CliConfiguration for BenchmarkCmd {
+impl CliConfiguration for PalletCmd {
 	fn shared_params(&self) -> &SharedParams {
 		&self.shared_params
 	}

substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs

@@ -0,0 +1,150 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod command;
+mod writer;
+
+use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
+use std::{fmt::Debug, path::PathBuf};
+
+// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
+// used like crate names with `_`
+fn parse_pallet_name(pallet: &str) -> String {
+	pallet.replace("-", "_")
+}
+
+/// Benchmark the extrinsic weight of FRAME Pallets.
+#[derive(Debug, clap::Parser)]
+pub struct PalletCmd {
+	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
+	#[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")]
+	pub pallet: Option<String>,
+
+	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
+	#[clap(short, long, required_unless_present = "list")]
+	pub extrinsic: Option<String>,
+
+	/// Select how many samples we should take across the variable components.
+	#[clap(short, long, default_value = "1")]
+	pub steps: u32,
+
+	/// Indicates lowest values for each of the component ranges.
+	#[clap(long = "low", use_value_delimiter = true)]
+	pub lowest_range_values: Vec<u32>,
+
+	/// Indicates highest values for each of the component ranges.
+	#[clap(long = "high", use_value_delimiter = true)]
+	pub highest_range_values: Vec<u32>,
+
+	/// Select how many repetitions of this benchmark should run from within the wasm.
+	#[clap(short, long, default_value = "1")]
+	pub repeat: u32,
+
+	/// Select how many repetitions of this benchmark should run from the client.
+	///
+	/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
+	#[clap(long, default_value = "1")]
+	pub external_repeat: u32,
+
+	/// Print the raw results in JSON format.
+	#[clap(long = "json")]
+	pub json_output: bool,
+
+	/// Write the raw results in JSON format into the given file.
+	#[clap(long, conflicts_with = "json-output")]
+	pub json_file: Option<PathBuf>,
+
+	/// Don't print the median-slopes linear regression analysis.
+	#[clap(long)]
+	pub no_median_slopes: bool,
+
+	/// Don't print the min-squares linear regression analysis.
+	#[clap(long)]
+	pub no_min_squares: bool,
+
+	/// Output the benchmarks to a Rust file at the given path.
+	#[clap(long)]
+	pub output: Option<PathBuf>,
+
+	/// Add a header file to your outputted benchmarks.
+	#[clap(long)]
+	pub header: Option<PathBuf>,
+
+	/// Path to Handlebars template file used for outputting benchmark results. (Optional)
+	#[clap(long)]
+	pub template: Option<PathBuf>,
+
+	/// Which analysis function to use when outputting benchmarks:
+	/// * min-squares (default)
+	/// * median-slopes
+	/// * max (max of min squares and median slopes for each value)
+	#[clap(long)]
+	pub output_analysis: Option<String>,
+
+	/// Set the heap pages while running benchmarks. If not set, the default value from the client
+	/// is used.
+	#[clap(long)]
+	pub heap_pages: Option<u64>,
+
+	/// Disable verification logic when running benchmarks.
+	#[clap(long)]
+	pub no_verify: bool,
+
+	/// Display and run extra benchmarks that would otherwise not be needed for weight
+	/// construction.
+	#[clap(long)]
+	pub extra: bool,
+
+	/// Estimate PoV size.
+	#[clap(long)]
+	pub record_proof: bool,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: sc_cli::SharedParams,
+
+	/// The execution strategy that should be used for benchmarks.
+	#[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)]
+	pub execution: Option<ExecutionStrategy>,
+
+	/// Method for executing Wasm runtime code.
+	#[clap(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = WasmExecutionMethod::variants(),
+		ignore_case = true,
+		default_value = DEFAULT_WASM_EXECUTION_METHOD,
+	)]
+	pub wasm_method: WasmExecutionMethod,
+
+	/// Limit the memory the database cache can use.
+	#[clap(long = "db-cache", value_name = "MiB", default_value = "1024")]
+	pub database_cache_size: u32,
+
+	/// List the benchmarks that match your query rather than running them.
+	///
+	/// When nothing is provided, we list all benchmarks.
+	#[clap(long)]
+	pub list: bool,
+
+	/// If enabled, the storage info is not displayed in the output next to the analysis.
+	///
+	/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
+	/// template for that purpose.
+	#[clap(long)]
+	pub no_storage_info: bool,
+}

substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs

@@ -26,7 +26,7 @@ use std::{
 use inflector::Inflector;
 use serde::Serialize;
 
-use crate::BenchmarkCmd;
+use crate::{shared::UnderscoreHelper, PalletCmd};
 use frame_benchmarking::{
 	Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector,
 	RegressionModel,
@@ -68,7 +68,7 @@ struct BenchmarkData {
 	comments: Vec<String>,
 }
 
-// This forwards some specific metadata from the `BenchmarkCmd`
+// This forwards some specific metadata from the `PalletCmd`
 #[derive(Serialize, Default, Debug, Clone)]
 struct CmdData {
 	steps: u32,
@@ -255,7 +255,7 @@ pub fn write_results(
 	batches: &[BenchmarkBatchSplitResults],
 	storage_info: &[StorageInfo],
 	path: &PathBuf,
-	cmd: &BenchmarkCmd,
+	cmd: &PalletCmd,
 ) -> Result<(), std::io::Error> {
 	// Use custom template if provided.
 	let template: String = match &cmd.template {
@@ -416,44 +416,6 @@ pub(crate) fn add_storage_comments(
 	}
 }
 
-// Add an underscore after every 3rd character, i.e. a separator for large numbers.
-fn underscore<Number>(i: Number) -> String
-where
-	Number: std::string::ToString,
-{
-	let mut s = String::new();
-	let i_str = i.to_string();
-	let a = i_str.chars().rev().enumerate();
-	for (idx, val) in a {
-		if idx != 0 && idx % 3 == 0 {
-			s.insert(0, '_');
-		}
-		s.insert(0, val);
-	}
-	s
-}
-
-// A Handlebars helper to add an underscore after every 3rd character,
-// i.e. a separator for large numbers.
-#[derive(Clone, Copy)]
-pub(crate) struct UnderscoreHelper;
-impl handlebars::HelperDef for UnderscoreHelper {
-	fn call<'reg: 'rc, 'rc>(
-		&self,
-		h: &handlebars::Helper,
-		_: &handlebars::Handlebars,
-		_: &handlebars::Context,
-		_rc: &mut handlebars::RenderContext,
-		out: &mut dyn handlebars::Output,
-	) -> handlebars::HelperResult {
-		use handlebars::JsonRender;
-		let param = h.param(0).unwrap();
-		let underscore_param = underscore(param.value().render());
-		out.write(&underscore_param)?;
-		Ok(())
-	}
-}
-
 // A helper to join a string of vectors.
 #[derive(Clone, Copy)]
 struct JoinHelper;

substrate/utils/frame/benchmarking-cli/src/shared/mod.rs

@@ -0,0 +1,65 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Code that is shared among all benchmarking sub-commands.
+
+pub mod record;
+pub mod stats;
+pub mod weight_params;
+
+pub use record::BenchRecord;
+pub use stats::{StatSelect, Stats};
+pub use weight_params::WeightParams;
+
+/// A Handlebars helper to add an underscore after every 3rd character,
+/// i.e. a separator for large numbers.
+#[derive(Clone, Copy)]
+pub struct UnderscoreHelper;
+
+impl handlebars::HelperDef for UnderscoreHelper {
+	fn call<'reg: 'rc, 'rc>(
+		&self,
+		h: &handlebars::Helper,
+		_: &handlebars::Handlebars,
+		_: &handlebars::Context,
+		_rc: &mut handlebars::RenderContext,
+		out: &mut dyn handlebars::Output,
+	) -> handlebars::HelperResult {
+		use handlebars::JsonRender;
+		let param = h.param(0).unwrap();
+		let underscore_param = underscore(param.value().render());
+		out.write(&underscore_param)?;
+		Ok(())
+	}
+}
+
+/// Add an underscore after every 3rd character, i.e. a separator for large numbers.
+fn underscore<Number>(i: Number) -> String
+where
+	Number: std::string::ToString,
+{
+	let mut s = String::new();
+	let i_str = i.to_string();
+	let a = i_str.chars().rev().enumerate();
+	for (idx, val) in a {
+		if idx != 0 && idx % 3 == 0 {
+			s.insert(0, '_');
+		}
+		s.insert(0, val);
+	}
+	s
+}

substrate/utils/frame/benchmarking-cli/src/shared/record.rs

@@ -0,0 +1,72 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`].
+
+use sc_cli::Result;
+use sc_service::Configuration;
+
+use log::info;
+use serde::Serialize;
+use std::{fs, path::PathBuf, time::Duration};
+
+use super::Stats;
+
+/// Raw output of a Storage benchmark.
+#[derive(Debug, Default, Clone, Serialize)]
+pub struct BenchRecord {
+	/// Multi-Map of value sizes and the time that it took to access them.
+	ns_per_size: Vec<(u64, u64)>,
+}
+
+impl BenchRecord {
+	/// Appends a new record. Uses safe casts.
+	pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
+		let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
+		let ns: u64 = d
+			.as_nanos()
+			.try_into()
+			.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
+		self.ns_per_size.push((size, ns));
+		Ok(())
+	}
+
+	/// Returns the statistics for *time* and *value size*.
+	pub fn calculate_stats(self) -> Result<(Stats, Stats)> {
+		let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
+		let size = Stats::new(&size)?;
+		let time = Stats::new(&time)?;
+		Ok((time, size)) // The swap of time/size here is intentional.
+	}
+
+	/// Unless a path is specified, saves the raw results in a json file in the current directory.
+	/// Prefixes it with the DB name and suffixed with `path_suffix`.
+	pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
+		let mut path = PathBuf::from(out_path);
+		if path.is_dir() || path.as_os_str().is_empty() {
+			path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
+			path.set_extension("json");
+		}
+
+		let json = serde_json::to_string_pretty(&self)
+			.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
+
+		fs::write(&path, json)?;
+		info!("Raw data written to {:?}", fs::canonicalize(&path)?);
+		Ok(())
+	}
+}

substrate/utils/frame/benchmarking-cli/src/shared/stats.rs

@@ -15,46 +15,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Calculates statistics and fills out the `weight.hbs` template.
+//! Handles statistics that were generated from benchmarking results and
+//! that can be used to fill out weight templates.
 
 use sc_cli::Result;
-use sc_service::Configuration;
 
-use log::info;
 use serde::Serialize;
-use std::{fmt, fs, path::PathBuf, result, str::FromStr, time::Duration};
-
-/// Raw output of a Storage benchmark.
-#[derive(Debug, Default, Clone, Serialize)]
-pub(crate) struct BenchRecord {
-	/// Multi-Map of value sizes and the time that it took to access them.
-	ns_per_size: Vec<(u64, u64)>,
-}
+use std::{fmt, result, str::FromStr};
 
 /// Various statistics that help to gauge the quality of the produced weights.
 /// Will be written to the weight file and printed to console.
 #[derive(Serialize, Default, Clone)]
-pub(crate) struct Stats {
+pub struct Stats {
 	/// Sum of all values.
-	pub(crate) sum: u64,
+	pub sum: u64,
 	/// Minimal observed value.
-	pub(crate) min: u64,
+	pub min: u64,
 	/// Maximal observed value.
-	pub(crate) max: u64,
+	pub max: u64,
 
 	/// Average of all values.
-	pub(crate) avg: u64,
+	pub avg: u64,
 	/// Median of all values.
-	pub(crate) median: u64,
+	pub median: u64,
 	/// Standard derivation of all values.
-	pub(crate) stddev: f64,
+	pub stddev: f64,
 
 	/// 99th percentile. At least 99% of all values are below this threshold.
-	pub(crate) p99: u64,
+	pub p99: u64,
 	/// 95th percentile. At least 95% of all values are below this threshold.
-	pub(crate) p95: u64,
+	pub p95: u64,
 	/// 75th percentile. At least 75% of all values are below this threshold.
-	pub(crate) p75: u64,
+	pub p75: u64,
 }
 
 /// Selects a specific field from a [`Stats`] object.
@@ -75,44 +67,6 @@ pub enum StatSelect {
 	P75Percentile,
 }
 
-impl BenchRecord {
-	/// Appends a new record. Uses safe casts.
-	pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
-		let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
-		let ns: u64 = d
-			.as_nanos()
-			.try_into()
-			.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
-		self.ns_per_size.push((size, ns));
-		Ok(())
-	}
-
-	/// Returns the statistics for *time* and *value size*.
-	pub(crate) fn calculate_stats(self) -> Result<(Stats, Stats)> {
-		let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
-		let size = Stats::new(&size)?;
-		let time = Stats::new(&time)?;
-		Ok((time, size)) // The swap of time/size here is intentional.
-	}
-
-	/// Unless a path is specified, saves the raw results in a json file in the current directory.
-	/// Prefixes it with the DB name and suffixed with `path_suffix`.
-	pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
-		let mut path = PathBuf::from(out_path);
-		if path.is_dir() || path.as_os_str().is_empty() {
-			path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
-			path.set_extension("json");
-		}
-
-		let json = serde_json::to_string_pretty(&self)
-			.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
-
-		fs::write(&path, json)?;
-		info!("Raw data written to {:?}", fs::canonicalize(&path)?);
-		Ok(())
-	}
-}
-
 impl Stats {
 	/// Calculates statistics and returns them.
 	pub fn new(xs: &Vec<u64>) -> Result<Self> {
@@ -137,7 +91,7 @@ impl Stats {
 	}
 
 	/// Returns the selected stat.
-	pub(crate) fn select(&self, s: StatSelect) -> u64 {
+	pub fn select(&self, s: StatSelect) -> u64 {
 		match s {
 			StatSelect::Maximum => self.max,
 			StatSelect::Average => self.avg,

substrate/utils/frame/benchmarking-cli/src/shared/weight_params.rs

@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Calculates a weight from the statistics of a benchmark result.
+//! Calculates a weight from the [`super::Stats`] of a benchmark result.
 
 use sc_cli::Result;
 
@@ -23,7 +23,7 @@ use clap::Args;
 use serde::Serialize;
 use std::path::PathBuf;
 
-use crate::storage::record::{StatSelect, Stats};
+use super::{StatSelect, Stats};
 
 /// Configures the weight generation.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
@@ -55,7 +55,7 @@ pub struct WeightParams {
 /// `weight_mul` and adding `weight_add`.
 /// Does not use safe casts and can overflow.
 impl WeightParams {
-	pub(crate) fn calc_weight(&self, stat: &Stats) -> Result<u64> {
+	pub fn calc_weight(&self, stat: &Stats) -> Result<u64> {
 		if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() {
 			return Err("invalid floating number for `weight_mul`".into())
 		}
@@ -68,7 +68,7 @@ impl WeightParams {
 #[cfg(test)]
 mod test_weight_params {
 	use super::WeightParams;
-	use crate::storage::record::{StatSelect, Stats};
+	use crate::shared::{StatSelect, Stats};
 
 	#[test]
 	fn calc_weight_works() {

substrate/utils/frame/benchmarking-cli/src/storage/cmd.rs

@@ -34,8 +34,9 @@ use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
 
 use super::template::TemplateData;
-use crate::post_processing::WeightParams;
-/// Benchmark the storage of a Substrate node with a live chain snapshot.
+use crate::shared::WeightParams;
+
+/// Benchmark the storage speed of a chain snapshot.
 #[derive(Debug, Parser)]
 pub struct StorageCmd {
 	#[allow(missing_docs)]
@@ -99,7 +100,7 @@ pub struct StorageParams {
 impl StorageCmd {
 	/// Calls into the Read and Write benchmarking functions.
 	/// Processes the output and writes it into files and stdout.
-	pub async fn run<Block, BA, C>(
+	pub fn run<Block, BA, C>(
 		&self,
 		cfg: Configuration,
 		client: Arc<C>,

substrate/utils/frame/benchmarking-cli/src/storage/mod.rs

@@ -17,7 +17,6 @@
 
 pub mod cmd;
 pub mod read;
-pub mod record;
 pub mod template;
 pub mod write;
 

substrate/utils/frame/benchmarking-cli/src/storage/read.rs

@@ -27,7 +27,8 @@ use log::info;
 use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
-use super::{cmd::StorageCmd, record::BenchRecord};
+use super::cmd::StorageCmd;
+use crate::shared::BenchRecord;
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to read a single Storage item.

substrate/utils/frame/benchmarking-cli/src/storage/template.rs

@@ -22,7 +22,8 @@ use log::info;
 use serde::Serialize;
 use std::{env, fs, path::PathBuf};
 
-use super::{cmd::StorageParams, record::Stats};
+use super::cmd::StorageParams;
+use crate::shared::{Stats, UnderscoreHelper};
 
 static VERSION: &'static str = env!("CARGO_PKG_VERSION");
 static TEMPLATE: &str = include_str!("./weights.hbs");
@@ -97,7 +98,7 @@ impl TemplateData {
 	pub fn write(&self, path: &Option<PathBuf>, hbs_template: &Option<PathBuf>) -> Result<()> {
 		let mut handlebars = handlebars::Handlebars::new();
 		// Format large integers with underscore.
-		handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper));
+		handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
 		// Don't HTML escape any characters.
 		handlebars.register_escape_fn(|s| -> String { s.to_string() });
 		// Use custom template if provided.

substrate/utils/frame/benchmarking-cli/src/storage/write.rs

@@ -31,7 +31,8 @@ use log::{info, trace};
 use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
-use super::{cmd::StorageCmd, record::BenchRecord};
+use super::cmd::StorageCmd;
+use crate::shared::BenchRecord;
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to write a single Storage item.