Add execution overhead benchmarking (#10977)

* Add benchmark-block

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove first approach

This reverts commit cf96a0a2307433f23187e77864de4a89ecbaef0a.

* Add block and extrinsic benchmarks

* Doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix template

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Beauty fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Check for non-empty chain

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add tests for Stats

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Apply suggestions from code review

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Push first version again

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Push first version again

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Cleanup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Cleanup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Cleanup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Beauty fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update utils/frame/benchmarking-cli/src/overhead/template.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Doc + Template fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Comment fix

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Pust merge fixup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fixup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move code to better place

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/Cargo.lock

@@ -2139,6 +2139,7 @@ dependencies = [
  "memory-db",
  "parity-scale-codec",
  "rand 0.8.4",
+ "sc-block-builder",
  "sc-cli",
  "sc-client-api",
  "sc-client-db",
@@ -2152,6 +2153,7 @@ dependencies = [
  "sp-core",
  "sp-database",
  "sp-externalities",
+ "sp-inherents",
  "sp-keystore",
  "sp-runtime",
  "sp-state-machine",

substrate/bin/node/cli/src/cli.rs

@@ -42,6 +42,13 @@ pub enum Subcommand {
 	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
+	/// Sub command for benchmarking the per-block and per-extrinsic execution overhead.
+	#[clap(
+		name = "benchmark-overhead",
+		about = "Benchmark the per-block and per-extrinsic execution overhead."
+	)]
+	BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd),
+
 	/// Sub command for benchmarking the storage speed.
 	#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
 	BenchmarkStorage(frame_benchmarking_cli::StorageCmd),

substrate/bin/node/cli/src/command.rs

@@ -18,10 +18,13 @@
 
 use crate::{chain_spec, service, service::new_partial, Cli, Subcommand};
 use node_executor::ExecutorDispatch;
-use node_runtime::{Block, RuntimeApi};
+use node_primitives::Block;
+use node_runtime::RuntimeApi;
 use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
 
+use std::sync::Arc;
+
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
 		"Substrate Node".into()
@@ -95,13 +98,21 @@ pub fn run() -> Result<()> {
 				You can enable it with `--features runtime-benchmarks`."
 					.into())
 			},
-		Some(Subcommand::BenchmarkStorage(cmd)) => {
+		Some(Subcommand::BenchmarkOverhead(cmd)) => {
-			if !cfg!(feature = "runtime-benchmarks") {
+			let runner = cli.create_runner(cmd)?;
-				return Err("Benchmarking wasn't enabled when building the node. \
+			runner.async_run(|mut config| {
-				You can enable it with `--features runtime-benchmarks`."
+				use super::command_helper::{inherent_data, ExtrinsicBuilder};
-					.into())
+				// We don't use the authority role since that would start producing blocks
-			}
+				// in the background which would mess with our benchmark.
+				config.role = sc_service::Role::Full;
 
+				let PartialComponents { client, task_manager, .. } = new_partial(&config)?;
+				let ext_builder = ExtrinsicBuilder::new(client.clone());
+
+				Ok((cmd.run(config, client, inherent_data()?, Arc::new(ext_builder)), task_manager))
+			})
+		},
+		Some(Subcommand::BenchmarkStorage(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			runner.async_run(|config| {
 				let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?;

substrate/bin/node/cli/src/command_helper.rs

@@ -0,0 +1,69 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Contains code to setup the command invocations in [`super::command`] which would
+//! otherwise bloat that module.
+
+use crate::service::{create_extrinsic, FullClient};
+
+use node_runtime::SystemCall;
+use sc_cli::Result;
+use sp_inherents::{InherentData, InherentDataProvider};
+use sp_keyring::Sr25519Keyring;
+use sp_runtime::OpaqueExtrinsic;
+
+use std::{sync::Arc, time::Duration};
+
+/// Generates extrinsics for the `benchmark-overhead` command.
+pub struct ExtrinsicBuilder {
+	client: Arc<FullClient>,
+}
+
+impl ExtrinsicBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>) -> Self {
+		Self { client }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder {
+	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_extrinsic(
+			self.client.as_ref(),
+			acc,
+			SystemCall::remark { remark: vec![] },
+			Some(nonce),
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
+/// Generates inherent data for the `benchmark-overhead` command.
+pub fn inherent_data() -> Result<InherentData> {
+	let mut inherent_data = InherentData::new();
+	let d = Duration::from_millis(0);
+	let timestamp = sp_timestamp::InherentDataProvider::new(d.into());
+
+	timestamp
+		.provide_inherent_data(&mut inherent_data)
+		.map_err(|e| format!("creating inherent data: {:?}", e))?;
+	Ok(inherent_data)
+}

substrate/bin/node/cli/src/lib.rs

@@ -38,6 +38,8 @@ pub mod service;
 mod cli;
 #[cfg(feature = "cli")]
 mod command;
+#[cfg(feature = "cli")]
+mod command_helper;
 
 #[cfg(feature = "cli")]
 pub use cli::*;

substrate/bin/node/cli/tests/benchmark_overhead_works.rs

@@ -0,0 +1,46 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::process::Command;
+use tempfile::tempdir;
+
+/// Tests that the `benchmark-overhead` command works for the substrate dev runtime.
+#[test]
+fn benchmark_overhead_works() {
+	let tmp_dir = tempdir().expect("could not create a temp dir");
+	let base_path = tmp_dir.path();
+
+	// Only put 10 extrinsics into the block otherwise it takes forever to build it
+	// especially for a non-release build.
+	let status = Command::new(cargo_bin("substrate"))
+		.args(&["benchmark-overhead", "--dev", "-d"])
+		.arg(base_path)
+		.arg("--weight-path")
+		.arg(base_path)
+		.args(["--warmup", "10", "--repeat", "10"])
+		.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
+		.args(["--max-ext-per-block", "10"])
+		.status()
+		.unwrap();
+	assert!(status.success());
+
+	// Weight files have been created.
+	assert!(base_path.join("block_weights.rs").exists());
+	assert!(base_path.join("extrinsic_weights.rs").exists());
+}

substrate/utils/frame/benchmarking-cli/Cargo.toml

@@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" }
 frame-support = { version = "4.0.0-dev", path = "../../../frame/support" }
 sp-core = { version = "6.0.0", path = "../../../primitives/core" }
+sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" }
 sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" }
 sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }
 sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" }
@@ -26,6 +27,7 @@ sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" }
 sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" }
 sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }
+sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" }
 sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" }
 sp-storage = { version = "6.0.0", path = "../../../primitives/storage" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -16,12 +16,15 @@
 // limitations under the License.
 
 mod command;
+pub mod overhead;
+mod post_processing;
 mod storage;
 mod writer;
 
 use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
 use std::{fmt::Debug, path::PathBuf};
 
+pub use overhead::{ExtrinsicBuilder, OverheadCmd};
 pub use storage::StorageCmd;
 
 // Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be

substrate/utils/frame/benchmarking-cli/src/overhead/bench.rs

@@ -0,0 +1,210 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the core benchmarking logic.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{Error, Result};
+use sc_client_api::Backend as ClientBackend;
+use sp_api::{ApiExt, BlockId, Core, ProvideRuntimeApi};
+use sp_blockchain::{
+	ApplyExtrinsicFailed::Validity,
+	Error::{ApplyExtrinsicFailed, RuntimeApiError},
+};
+use sp_runtime::{
+	traits::{Block as BlockT, Zero},
+	transaction_validity::{InvalidTransaction, TransactionValidityError},
+	OpaqueExtrinsic,
+};
+
+use clap::Args;
+use log::info;
+use serde::Serialize;
+use std::{marker::PhantomData, sync::Arc, time::Instant};
+
+use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats};
+
+/// Parameters to configure an *overhead* benchmark.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct BenchmarkParams {
+	/// Rounds of warmups before measuring.
+	#[clap(long, default_value = "100")]
+	pub warmup: u32,
+
+	/// How many times the benchmark should be repeated.
+	#[clap(long, default_value = "1000")]
+	pub repeat: u32,
+
+	/// Maximal number of extrinsics that should be put into a block.
+	///
+	/// Only useful for debugging.
+	#[clap(long)]
+	pub max_ext_per_block: Option<u32>,
+}
+
+/// The results of multiple runs in nano seconds.
+pub(crate) type BenchRecord = Vec<u64>;
+
+/// Type of a benchmark.
+#[derive(Serialize, Clone, PartialEq, Copy)]
+pub(crate) enum BenchmarkType {
+	/// Measure the per-extrinsic execution overhead.
+	Extrinsic,
+	/// Measure the per-block execution overhead.
+	Block,
+}
+
+/// Holds all objects needed to run the *overhead* benchmarks.
+pub(crate) struct Benchmark<Block, BA, C> {
+	client: Arc<C>,
+	params: BenchmarkParams,
+	inherent_data: sp_inherents::InherentData,
+	ext_builder: Arc<dyn ExtrinsicBuilder>,
+	_p: PhantomData<(Block, BA)>,
+}
+
+impl<Block, BA, C> Benchmark<Block, BA, C>
+where
+	Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+	BA: ClientBackend<Block>,
+	C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+	C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+{
+	/// Create a new [`Self`] from the arguments.
+	pub fn new(
+		client: Arc<C>,
+		params: BenchmarkParams,
+		inherent_data: sp_inherents::InherentData,
+		ext_builder: Arc<dyn ExtrinsicBuilder>,
+	) -> Self {
+		Self { client, params, inherent_data, ext_builder, _p: PhantomData }
+	}
+
+	/// Run the specified benchmark.
+	pub fn bench(&self, bench_type: BenchmarkType) -> Result<Stats> {
+		let (block, num_ext) = self.build_block(bench_type)?;
+		let record = self.measure_block(&block, num_ext, bench_type)?;
+		Stats::new(&record)
+	}
+
+	/// Builds a block for the given benchmark type.
+	///
+	/// Returns the block and the number of extrinsics in the block
+	/// that are not inherents.
+	fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> {
+		let mut builder = self.client.new_block(Default::default())?;
+		// Create and insert the inherents.
+		let inherents = builder.create_inherents(self.inherent_data.clone())?;
+		for inherent in inherents {
+			builder.push(inherent)?;
+		}
+
+		// Return early if we just want a block with inherents and no additional extrinsics.
+		if bench_type == BenchmarkType::Block {
+			return Ok((builder.build()?.block, 0))
+		}
+
+		// Put as many extrinsics into the block as possible and count them.
+		info!("Building block, this takes some time...");
+		let mut num_ext = 0;
+		for nonce in 0..self.max_ext_per_block() {
+			let ext = self.ext_builder.remark(nonce)?;
+			match builder.push(ext.clone()) {
+				Ok(()) => {},
+				Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
+					InvalidTransaction::ExhaustsResources,
+				)))) => break, // Block is full
+				Err(e) => return Err(Error::Client(e)),
+			}
+			num_ext += 1;
+		}
+		if num_ext == 0 {
+			return Err("A Block must hold at least one extrinsic".into())
+		}
+		info!("Extrinsics per block: {}", num_ext);
+		let block = builder.build()?.block;
+
+		Ok((block, num_ext))
+	}
+
+	/// Measures the time that it take to execute a block or an extrinsic.
+	fn measure_block(
+		&self,
+		block: &Block,
+		num_ext: u64,
+		bench_type: BenchmarkType,
+	) -> Result<BenchRecord> {
+		let mut record = BenchRecord::new();
+		if bench_type == BenchmarkType::Extrinsic && num_ext == 0 {
+			return Err("Cannot measure the extrinsic time of an empty block".into())
+		}
+		let genesis = BlockId::Number(Zero::zero());
+
+		info!("Running {} warmups...", self.params.warmup);
+		for _ in 0..self.params.warmup {
+			self.client
+				.runtime_api()
+				.execute_block(&genesis, block.clone())
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+		}
+
+		info!("Executing block {} times", self.params.repeat);
+		// Interesting part here:
+		// Execute a block multiple times and record each execution time.
+		for _ in 0..self.params.repeat {
+			let block = block.clone();
+			let runtime_api = self.client.runtime_api();
+			let start = Instant::now();
+
+			runtime_api
+				.execute_block(&genesis, block)
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+
+			let elapsed = start.elapsed().as_nanos();
+			if bench_type == BenchmarkType::Extrinsic {
+				// Checked for non-zero div above.
+				record.push((elapsed as f64 / num_ext as f64).ceil() as u64);
+			} else {
+				record.push(elapsed as u64);
+			}
+		}
+
+		Ok(record)
+	}
+
+	fn max_ext_per_block(&self) -> u32 {
+		self.params.max_ext_per_block.unwrap_or(u32::MAX)
+	}
+}
+
+impl BenchmarkType {
+	/// Short name of the benchmark type.
+	pub(crate) fn short_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "extrinsic",
+			Self::Block => "block",
+		}
+	}
+
+	/// Long name of the benchmark type.
+	pub(crate) fn long_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "ExtrinsicBase",
+			Self::Block => "BlockExecution",
+		}
+	}
+}

substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs

@@ -0,0 +1,118 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the [`OverheadCmd`] as entry point for the CLI to execute
+//! the *overhead* benchmarks.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{CliConfiguration, Result, SharedParams};
+use sc_client_api::Backend as ClientBackend;
+use sc_service::Configuration;
+use sp_api::{ApiExt, ProvideRuntimeApi};
+use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic};
+
+use clap::{Args, Parser};
+use log::info;
+use serde::Serialize;
+use std::{fmt::Debug, sync::Arc};
+
+use crate::{
+	overhead::{
+		bench::{Benchmark, BenchmarkParams, BenchmarkType},
+		template::TemplateData,
+	},
+	post_processing::WeightParams,
+};
+
+/// Benchmarks the per-block and per-extrinsic execution overhead.
+#[derive(Debug, Parser)]
+pub struct OverheadCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub params: OverheadParams,
+}
+
+/// Configures the benchmark, the post-processing and weight generation.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct OverheadParams {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub weight: WeightParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub bench: BenchmarkParams,
+}
+
+/// Used by the benchmark to build signed extrinsics.
+///
+/// The built extrinsics only need to be valid in the first block
+/// who's parent block is the genesis block.
+pub trait ExtrinsicBuilder {
+	/// Build a `System::remark` extrinsic.
+	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str>;
+}
+
+impl OverheadCmd {
+	/// Measures the per-block and per-extrinsic execution overhead.
+	///
+	/// Writes the results to console and into two instances of the
+	/// `weights.hbs` template, one for each benchmark.
+	pub async fn run<Block, BA, C>(
+		&self,
+		cfg: Configuration,
+		client: Arc<C>,
+		inherent_data: sp_inherents::InherentData,
+		ext_builder: Arc<dyn ExtrinsicBuilder>,
+	) -> Result<()>
+	where
+		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+		BA: ClientBackend<Block>,
+		C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+		C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+	{
+		let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder);
+
+		// per-block execution overhead
+		{
+			let stats = bench.bench(BenchmarkType::Block)?;
+			info!("Per-block execution overhead [ns]:\n{:?}", stats);
+			let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?;
+			template.write(&self.params.weight.weight_path)?;
+		}
+		// per-extrinsic execution overhead
+		{
+			let stats = bench.bench(BenchmarkType::Extrinsic)?;
+			info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats);
+			let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?;
+			template.write(&self.params.weight.weight_path)?;
+		}
+
+		Ok(())
+	}
+}
+
+// Boilerplate
+impl CliConfiguration for OverheadCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+}

substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs

@@ -0,0 +1,22 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod bench;
+pub mod cmd;
+mod template;
+
+pub use cmd::{ExtrinsicBuilder, OverheadCmd};

substrate/utils/frame/benchmarking-cli/src/overhead/template.rs

@@ -0,0 +1,110 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Converts a benchmark result into [`TemplateData`] and writes
+//! it into the `weights.hbs` template.
+
+use sc_cli::Result;
+use sc_service::Configuration;
+
+use handlebars::Handlebars;
+use log::info;
+use serde::Serialize;
+use std::{env, fs, path::PathBuf};
+
+use crate::{
+	overhead::{bench::BenchmarkType, cmd::OverheadParams},
+	storage::record::Stats,
+};
+
+static VERSION: &'static str = env!("CARGO_PKG_VERSION");
+static TEMPLATE: &str = include_str!("./weights.hbs");
+
+/// Data consumed by Handlebar to fill out the `weights.hbs` template.
+#[derive(Serialize, Debug, Clone)]
+pub(crate) struct TemplateData {
+	/// Short name of the benchmark. Can be "block" or "extrinsic".
+	long_name: String,
+	/// Long name of the benchmark. Can be "BlockExecution" or "ExtrinsicBase".
+	short_name: String,
+	/// Name of the runtime. Taken from the chain spec.
+	runtime_name: String,
+	/// Version of the benchmarking CLI used.
+	version: String,
+	/// Date that the template was filled out.
+	date: String,
+	/// Command line arguments that were passed to the CLI.
+	args: Vec<String>,
+	/// Params of the executed command.
+	params: OverheadParams,
+	/// Stats about the benchmark result.
+	stats: Stats,
+	/// The resulting weight in ns.
+	weight: u64,
+}
+
+impl TemplateData {
+	/// Returns a new [`Self`] from the given params.
+	pub(crate) fn new(
+		t: BenchmarkType,
+		cfg: &Configuration,
+		params: &OverheadParams,
+		stats: &Stats,
+	) -> Result<Self> {
+		let weight = params.weight.calc_weight(stats)?;
+
+		Ok(TemplateData {
+			short_name: t.short_name().into(),
+			long_name: t.long_name().into(),
+			runtime_name: cfg.chain_spec.name().into(),
+			version: VERSION.into(),
+			date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(),
+			args: env::args().collect::<Vec<String>>(),
+			params: params.clone(),
+			stats: stats.clone(),
+			weight,
+		})
+	}
+
+	/// Fill out the `weights.hbs` HBS template with its own data.
+	/// Writes the result to `path` which can be a directory or a file.
+	pub fn write(&self, path: &Option<PathBuf>) -> Result<()> {
+		let mut handlebars = Handlebars::new();
+		// Format large integers with underscores.
+		handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper));
+		// Don't HTML escape any characters.
+		handlebars.register_escape_fn(|s| -> String { s.to_string() });
+
+		let out_path = self.build_path(path)?;
+		let mut fd = fs::File::create(&out_path)?;
+		info!("Writing weights to {:?}", fs::canonicalize(&out_path)?);
+		handlebars
+			.render_template_to_write(&TEMPLATE, &self, &mut fd)
+			.map_err(|e| format!("HBS template write: {:?}", e).into())
+	}
+
+	/// Build a path for the weight file.
+	fn build_path(&self, weight_out: &Option<PathBuf>) -> Result<PathBuf> {
+		let mut path = weight_out.clone().unwrap_or(PathBuf::from("."));
+
+		if !path.is_dir() {
+			return Err("Need directory as --weight-path".into())
+		}
+		path.push(format!("{}_weights.rs", self.short_name));
+		Ok(path)
+	}
+}

substrate/utils/frame/benchmarking-cli/src/overhead/weights.hbs

@@ -0,0 +1,92 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}}
+//! DATE: {{date}}
+//!
+//! SHORT-NAME: `{{short_name}}`, LONG-NAME: `{{long_name}}`, RUNTIME: `{{runtime_name}}`
+//! WARMUPS: `{{params.bench.warmup}}`, REPEAT: `{{params.bench.repeat}}`
+//! WEIGHT-PATH: `{{params.weight.weight_path}}`
+//! WEIGHT-METRIC: `{{params.weight.weight_metric}}`, WEIGHT-MUL: `{{params.weight.weight_mul}}`, WEIGHT-ADD: `{{params.weight.weight_add}}`
+
+// Executed Command:
+{{#each args as |arg|}}
+//   {{arg}}
+{{/each}}
+
+use frame_support::{
+	parameter_types,
+	weights::{constants::WEIGHT_PER_NANOS, Weight},
+};
+
+parameter_types! {
+	{{#if (eq short_name "block")}}
+	/// Time to execute an empty block.
+	{{else}}
+	/// Time to execute a NO-OP extrinsic eg. `System::remark`.
+	{{/if}}
+	/// Calculated by multiplying the *{{params.weight.weight_metric}}* with `{{params.weight.weight_mul}}` and adding `{{params.weight.weight_add}}`.
+	///
+	/// Stats [ns]:
+	///   Min, Max: {{underscore stats.min}}, {{underscore stats.max}}
+	///   Average:  {{underscore stats.avg}}
+	///   Median:   {{underscore stats.median}}
+	///   StdDev:   {{stats.stddev}}
+	///
+	/// Percentiles [ns]:
+	///   99th: {{underscore stats.p99}}
+	///   95th: {{underscore stats.p95}}
+	///   75th: {{underscore stats.p75}}
+	pub const {{long_name}}Weight: Weight = {{underscore weight}} * WEIGHT_PER_NANOS;
+}
+
+#[cfg(test)]
+mod test_weights {
+	use frame_support::weights::constants;
+
+	/// Checks that the weight exists and is sane.
+	// NOTE: If this test fails but you are sure that the generated values are fine,
+	// you can delete it.
+	#[test]
+	fn sane() {
+		let w = super::{{long_name}}Weight::get();
+
+		{{#if (eq short_name "block")}}			
+		// At least 100 µs.
+		assert!(
+			w >= 100 * constants::WEIGHT_PER_MICROS,
+			"Weight should be at least 100 µs."
+		);
+		// At most 50 ms.
+		assert!(
+			w <= 50 * constants::WEIGHT_PER_MILLIS,
+			"Weight should be at most 50 ms."
+		);
+		{{else}}
+		// At least 10 µs.
+		assert!(
+			w >= 10 * constants::WEIGHT_PER_MICROS,
+			"Weight should be at least 10 µs."
+		);
+		// At most 1 ms.
+		assert!(
+			w <= constants::WEIGHT_PER_MILLIS,
+			"Weight should be at most 1 ms."
+		);
+		{{/if}}
+	}
+}

substrate/utils/frame/benchmarking-cli/src/post_processing/mod.rs

@@ -0,0 +1,95 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Calculates a weight from the statistics of a benchmark result.
+
+use sc_cli::Result;
+
+use clap::Args;
+use serde::Serialize;
+use std::path::PathBuf;
+
+use crate::storage::record::{StatSelect, Stats};
+
+/// Configures the weight generation.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct WeightParams {
+	/// File or directory to write the *weight* files to.
+	///
+	/// For Substrate this should be `frame/support/src/weights`.
+	#[clap(long)]
+	pub weight_path: Option<PathBuf>,
+
+	/// Select a specific metric to calculate the final weight output.
+	#[clap(long = "metric", default_value = "average")]
+	pub weight_metric: StatSelect,
+
+	/// Multiply the resulting weight with the given factor. Must be positive.
+	///
+	/// Is applied before `weight_add`.
+	#[clap(long = "mul", default_value = "1")]
+	pub weight_mul: f64,
+
+	/// Add the given offset to the resulting weight.
+	///
+	/// Is applied after `weight_mul`.
+	#[clap(long = "add", default_value = "0")]
+	pub weight_add: u64,
+}
+
+/// Calculates the final weight by multiplying the selected metric with
+/// `weight_mul` and adding `weight_add`.
+/// Does not use safe casts and can overflow.
+impl WeightParams {
+	pub(crate) fn calc_weight(&self, stat: &Stats) -> Result<u64> {
+		if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() {
+			return Err("invalid floating number for `weight_mul`".into())
+		}
+		let s = stat.select(self.weight_metric) as f64;
+		let w = s.mul_add(self.weight_mul, self.weight_add as f64).ceil();
+		Ok(w as u64) // No safe cast here since there is no `From<f64>` for `u64`.
+	}
+}
+
+#[cfg(test)]
+mod test_weight_params {
+	use super::WeightParams;
+	use crate::storage::record::{StatSelect, Stats};
+
+	#[test]
+	fn calc_weight_works() {
+		let stats = Stats { avg: 113, ..Default::default() };
+		let params = WeightParams {
+			weight_metric: StatSelect::Average,
+			weight_mul: 0.75,
+			weight_add: 3,
+			..Default::default()
+		};
+
+		let want = (113.0f64 * 0.75 + 3.0).ceil() as u64; // Ceil for overestimation.
+		let got = params.calc_weight(&stats).unwrap();
+		assert_eq!(want, got);
+	}
+
+	#[test]
+	fn calc_weight_detects_negative_mul() {
+		let stats = Stats::default();
+		let params = WeightParams { weight_mul: -0.75, ..Default::default() };
+
+		assert!(params.calc_weight(&stats).is_err());
+	}
+}

substrate/utils/frame/benchmarking-cli/src/storage/cmd.rs

@@ -33,8 +33,8 @@ use serde::Serialize;
 use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
 
-use super::{record::StatSelect, template::TemplateData};
+use super::template::TemplateData;
-
+use crate::post_processing::WeightParams;
 /// Benchmark the storage of a Substrate node with a live chain snapshot.
 #[derive(Debug, Parser)]
 pub struct StorageCmd {
@@ -58,24 +58,9 @@ pub struct StorageCmd {
 /// Parameters for modifying the benchmark behaviour and the post processing of the results.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
 pub struct StorageParams {
-	/// Path to write the *weight* file to. Can be a file or directory.
+	#[allow(missing_docs)]
-	/// For substrate this should be `frame/support/src/weights`.
+	#[clap(flatten)]
-	#[clap(long)]
+	pub weight_params: WeightParams,
-	pub weight_path: Option<PathBuf>,
-
-	/// Select a specific metric to calculate the final weight output.
-	#[clap(long = "metric", default_value = "average")]
-	pub weight_metric: StatSelect,
-
-	/// Multiply the resulting weight with the given factor. Must be positive.
-	/// Is calculated before `weight_add`.
-	#[clap(long = "mul", default_value = "1")]
-	pub weight_mul: f64,
-
-	/// Add the given offset to the resulting weight.
-	/// Is calculated after `weight_mul`.
-	#[clap(long = "add", default_value = "0")]
-	pub weight_add: u64,
 
 	/// Skip the `read` benchmark.
 	#[clap(long)]
@@ -153,7 +138,7 @@ impl StorageCmd {
 			template.set_stats(None, Some(stats))?;
 		}
 
-		template.write(&self.params.weight_path, &self.params.template_path)
+		template.write(&self.params.weight_params.weight_path, &self.params.template_path)
 	}
 
 	/// Returns the specified state version.

substrate/utils/frame/benchmarking-cli/src/storage/record.rs

@@ -36,25 +36,25 @@ pub(crate) struct BenchRecord {
 #[derive(Serialize, Default, Clone)]
 pub(crate) struct Stats {
 	/// Sum of all values.
-	sum: u64,
+	pub(crate) sum: u64,
 	/// Minimal observed value.
-	min: u64,
+	pub(crate) min: u64,
 	/// Maximal observed value.
-	max: u64,
+	pub(crate) max: u64,
 
 	/// Average of all values.
-	avg: u64,
+	pub(crate) avg: u64,
 	/// Median of all values.
-	median: u64,
+	pub(crate) median: u64,
 	/// Standard derivation of all values.
-	stddev: f64,
+	pub(crate) stddev: f64,
 
 	/// 99th percentile. At least 99% of all values are below this threshold.
-	p99: u64,
+	pub(crate) p99: u64,
 	/// 95th percentile. At least 95% of all values are below this threshold.
-	p95: u64,
+	pub(crate) p95: u64,
 	/// 75th percentile. At least 75% of all values are below this threshold.
-	p75: u64,
+	pub(crate) p75: u64,
 }
 
 /// Selects a specific field from a [`Stats`] object.
@@ -159,8 +159,8 @@ impl Stats {
 	/// This is best effort since it ignores the interpolation case.
 	fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
 		xs.sort();
-		let index = (xs.len() as f64 * p).ceil() as usize;
+		let index = (xs.len() as f64 * p).ceil() as usize - 1;
-		xs[index]
+		xs[index.clamp(0, xs.len() - 1)]
 	}
 }
 
@@ -195,3 +195,40 @@ impl FromStr for StatSelect {
 		}
 	}
 }
+
+#[cfg(test)]
+mod test_stats {
+	use super::Stats;
+	use rand::{seq::SliceRandom, thread_rng};
+
+	#[test]
+	fn stats_correct() {
+		let mut data: Vec<u64> = (1..=100).collect();
+		data.shuffle(&mut thread_rng());
+		let stats = Stats::new(&data).unwrap();
+
+		assert_eq!(stats.sum, 5050);
+		assert_eq!(stats.min, 1);
+		assert_eq!(stats.max, 100);
+
+		assert_eq!(stats.avg, 50);
+		assert_eq!(stats.median, 50); // 50.5 to be exact.
+		assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
+
+		assert_eq!(stats.p99, 99);
+		assert_eq!(stats.p95, 95);
+		assert_eq!(stats.p75, 75);
+	}
+
+	#[test]
+	fn no_panic_short_lengths() {
+		// Empty input does error.
+		assert!(Stats::new(&vec![]).is_err());
+
+		// Different small input lengths are fine.
+		for l in 1..10 {
+			let data = (0..=l).collect();
+			assert!(Stats::new(&data).is_ok());
+		}
+	}
+}

substrate/utils/frame/benchmarking-cli/src/storage/template.rs

@@ -77,11 +77,11 @@ impl TemplateData {
 		write: Option<(Stats, Stats)>,
 	) -> Result<()> {
 		if let Some(read) = read {
-			self.read_weight = calc_weight(&read.0, &self.params)?;
+			self.read_weight = self.params.weight_params.calc_weight(&read.0)?;
 			self.read = Some(read);
 		}
 		if let Some(write) = write {
-			self.write_weight = calc_weight(&write.0, &self.params)?;
+			self.write_weight = self.params.weight_params.calc_weight(&write.0)?;
 			self.write = Some(write);
 		}
 		Ok(())
@@ -130,15 +130,3 @@ impl TemplateData {
 		path
 	}
 }
-
-/// Calculates the final weight by multiplying the selected metric with
-/// `mul` and adding `add`.
-/// Does not use safe casts and can overflow.
-fn calc_weight(stat: &Stats, params: &StorageParams) -> Result<u64> {
-	if params.weight_mul.is_sign_negative() || !params.weight_mul.is_normal() {
-		return Err("invalid floating number for `weight_mul`".into())
-	}
-	let s = stat.select(params.weight_metric) as f64;
-	let w = s.mul_add(params.weight_mul, params.weight_add as f64).ceil();
-	Ok(w as u64) // No safe cast here since there is no `From<f64>` for `u64`.
-}