Add benchmark extrinsic command (#11456)

* Benchmark extrinsic

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Reduce warmup and repeat

Running this 1000 times with a full block takes ~33 minutes 🙈.
Reducing it to ~3 minutes per default.

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Make ExistentialDeposit public

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add 'bechmark extrinsic' command

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add --list and cleanup

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Unrelated Clippy

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Clippy

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix tests and doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Move implementations up + fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Dont use parameter_types macro

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Cache to_lowercase() call

The .to_lowercase() on the builder is actually not needes
since its already documented to only return lower case.

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Spelling

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use correct nightly for fmt...

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Rename ED

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix compile

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Subtract block base weight

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Use tmp folder for test

This should already be the case since --dev is passed but
somehow not...

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

substrate/bin/node-template/node/src/benchmarking.rs

@@ -16,13 +16,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-//! Contains code to setup the command invocations in [`super::command`] which would
-//! otherwise bloat that module.
+//! Setup code for [`super::command`] which would otherwise bloat that module.
+//!
+//! Should only be used for benchmarking as it may break in other contexts.
 
 use crate::service::FullClient;
 
 use node_template_runtime as runtime;
-use runtime::SystemCall;
+use runtime::{AccountId, Balance, BalancesCall, SystemCall};
 use sc_cli::Result;
 use sc_client_api::BlockBackend;
 use sp_core::{Encode, Pair};
@@ -35,19 +36,27 @@ use std::{sync::Arc, time::Duration};
 /// Generates extrinsics for the `benchmark overhead` command.
 ///
 /// Note: Should only be used for benchmarking.
-pub struct BenchmarkExtrinsicBuilder {
+pub struct RemarkBuilder {
 	client: Arc<FullClient>,
 }
 
-impl BenchmarkExtrinsicBuilder {
+impl RemarkBuilder {
 	/// Creates a new [`Self`] from the given client.
 	pub fn new(client: Arc<FullClient>) -> Self {
 		Self { client }
 	}
 }
 
-impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
-	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder {
+	fn pallet(&self) -> &str {
+		"system"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"remark"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
 		let acc = Sr25519Keyring::Bob.pair();
 		let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic(
 			self.client.as_ref(),
@@ -61,6 +70,49 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
 	}
 }
 
+/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct TransferKeepAliveBuilder {
+	client: Arc<FullClient>,
+	dest: AccountId,
+	value: Balance,
+}
+
+impl TransferKeepAliveBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>, dest: AccountId, value: Balance) -> Self {
+		Self { client, dest, value }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder {
+	fn pallet(&self) -> &str {
+		"balances"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"transfer_keep_alive"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic(
+			self.client.as_ref(),
+			acc,
+			BalancesCall::transfer_keep_alive {
+				dest: self.dest.clone().into(),
+				value: self.value.into(),
+			}
+			.into(),
+			nonce,
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
 /// Create a transaction using the given `call`.
 ///
 /// Note: Should only be used for benchmarking.

substrate/bin/node-template/node/src/command.rs

@@ -1,14 +1,14 @@
 use crate::{
+	benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder},
 	chain_spec,
 	cli::{Cli, Subcommand},
-	command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder},
 	service,
 };
-use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE};
-use node_template_runtime::Block;
+use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE};
+use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT};
 use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
-use std::sync::Arc;
+use sp_keyring::Sr25519Keyring;
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -137,9 +137,23 @@ pub fn run() -> sc_cli::Result<()> {
 					},
 					BenchmarkCmd::Overhead(cmd) => {
 						let PartialComponents { client, .. } = service::new_partial(&config)?;
-						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+						let ext_builder = RemarkBuilder::new(client.clone());
 
-						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+						cmd.run(config, client, inherent_benchmark_data()?, &ext_builder)
+					},
+					BenchmarkCmd::Extrinsic(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
+						// Register the *Remark* and *TKA* builders.
+						let ext_factory = ExtrinsicFactory(vec![
+							Box::new(RemarkBuilder::new(client.clone())),
+							Box::new(TransferKeepAliveBuilder::new(
+								client.clone(),
+								Sr25519Keyring::Alice.to_account_id(),
+								EXISTENTIAL_DEPOSIT,
+							)),
+						]);
+
+						cmd.run(client, inherent_benchmark_data()?, &ext_factory)
 					},
 					BenchmarkCmd::Machine(cmd) =>
 						cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()),

substrate/bin/node-template/node/src/main.rs

@@ -4,9 +4,9 @@
 mod chain_spec;
 #[macro_use]
 mod service;
+mod benchmarking;
 mod cli;
 mod command;
-mod command_helper;
 mod rpc;
 
 fn main() -> sc_cli::Result<()> {

substrate/bin/node-template/runtime/src/lib.rs

@@ -234,6 +234,9 @@ impl pallet_timestamp::Config for Runtime {
 	type WeightInfo = ();
 }
 
+/// Existential deposit.
+pub const EXISTENTIAL_DEPOSIT: u128 = 500;
+
 impl pallet_balances::Config for Runtime {
 	type MaxLocks = ConstU32<50>;
 	type MaxReserves = ();
@@ -243,7 +246,7 @@ impl pallet_balances::Config for Runtime {
 	/// The ubiquitous event type.
 	type Event = Event;
 	type DustRemoval = ();
-	type ExistentialDeposit = ConstU128<500>;
+	type ExistentialDeposit = ConstU128<EXISTENTIAL_DEPOSIT>;
 	type AccountStore = System;
 	type WeightInfo = pallet_balances::weights::SubstrateWeight<Runtime>;
 }

substrate/bin/node/cli/src/benchmarking.rs

@@ -16,12 +16,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-//! Contains code to setup the command invocations in [`super::command`] which would
-//! otherwise bloat that module.
+//! Setup code for [`super::command`] which would otherwise bloat that module.
+//!
+//! Should only be used for benchmarking as it may break in other contexts.
 
 use crate::service::{create_extrinsic, FullClient};
 
-use node_runtime::SystemCall;
+use node_primitives::{AccountId, Balance};
+use node_runtime::{BalancesCall, SystemCall};
 use sc_cli::Result;
 use sp_inherents::{InherentData, InherentDataProvider};
 use sp_keyring::Sr25519Keyring;
@@ -29,20 +31,30 @@ use sp_runtime::OpaqueExtrinsic;
 
 use std::{sync::Arc, time::Duration};
 
-/// Generates extrinsics for the `benchmark overhead` command.
-pub struct BenchmarkExtrinsicBuilder {
+/// Generates `System::Remark` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct RemarkBuilder {
 	client: Arc<FullClient>,
 }
 
-impl BenchmarkExtrinsicBuilder {
+impl RemarkBuilder {
 	/// Creates a new [`Self`] from the given client.
 	pub fn new(client: Arc<FullClient>) -> Self {
 		Self { client }
 	}
 }
 
-impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
-	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder {
+	fn pallet(&self) -> &str {
+		"system"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"remark"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
 		let acc = Sr25519Keyring::Bob.pair();
 		let extrinsic: OpaqueExtrinsic = create_extrinsic(
 			self.client.as_ref(),
@@ -56,6 +68,48 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
 	}
 }
 
+/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct TransferKeepAliveBuilder {
+	client: Arc<FullClient>,
+	dest: AccountId,
+	value: Balance,
+}
+
+impl TransferKeepAliveBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>, dest: AccountId, value: Balance) -> Self {
+		Self { client, dest, value }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder {
+	fn pallet(&self) -> &str {
+		"balances"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"transfer_keep_alive"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_extrinsic(
+			self.client.as_ref(),
+			acc,
+			BalancesCall::transfer_keep_alive {
+				dest: self.dest.clone().into(),
+				value: self.value.into(),
+			},
+			Some(nonce),
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
 /// Generates inherent data for the `benchmark overhead` command.
 pub fn inherent_benchmark_data() -> Result<InherentData> {
 	let mut inherent_data = InherentData::new();

substrate/bin/node/cli/src/command.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder};
+use super::benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder};
 use crate::{
 	chain_spec, service,
 	service::{new_partial, FullClient},
@@ -25,9 +25,10 @@ use crate::{
 use frame_benchmarking_cli::*;
 use node_executor::ExecutorDispatch;
 use node_primitives::Block;
-use node_runtime::RuntimeApi;
+use node_runtime::{ExistentialDeposit, RuntimeApi};
 use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
+use sp_keyring::Sr25519Keyring;
 
 use std::sync::Arc;
 
@@ -126,9 +127,23 @@ pub fn run() -> Result<()> {
 					},
 					BenchmarkCmd::Overhead(cmd) => {
 						let PartialComponents { client, .. } = new_partial(&config)?;
-						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+						let ext_builder = RemarkBuilder::new(client.clone());
 
-						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+						cmd.run(config, client, inherent_benchmark_data()?, &ext_builder)
+					},
+					BenchmarkCmd::Extrinsic(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
+						// Register the *Remark* and *TKA* builders.
+						let ext_factory = ExtrinsicFactory(vec![
+							Box::new(RemarkBuilder::new(client.clone())),
+							Box::new(TransferKeepAliveBuilder::new(
+								client.clone(),
+								Sr25519Keyring::Alice.to_account_id(),
+								ExistentialDeposit::get(),
+							)),
+						]);
+
+						cmd.run(client, inherent_benchmark_data()?, &ext_factory)
 					},
 					BenchmarkCmd::Machine(cmd) =>
 						cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()),

substrate/bin/node/cli/src/lib.rs

@@ -35,11 +35,11 @@ pub mod chain_spec;
 #[macro_use]
 pub mod service;
 #[cfg(feature = "cli")]
+mod benchmarking;
+#[cfg(feature = "cli")]
 mod cli;
 #[cfg(feature = "cli")]
 mod command;
-#[cfg(feature = "cli")]
-mod command_helper;
 
 #[cfg(feature = "cli")]
 pub use cli::*;

substrate/bin/node/cli/tests/benchmark_extrinsic_works.rs

@@ -0,0 +1,46 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::process::Command;
+use tempfile::tempdir;
+
+/// Tests that the `benchmark extrinsic` command works for
+/// remark and transfer_keep_alive within the substrate dev runtime.
+#[test]
+fn benchmark_extrinsic_works() {
+	benchmark_extrinsic("system", "remark");
+	benchmark_extrinsic("balances", "transfer_keep_alive");
+}
+
+/// Checks that the `benchmark extrinsic` command works for the given pallet and extrinsic.
+fn benchmark_extrinsic(pallet: &str, extrinsic: &str) {
+	let base_dir = tempdir().expect("could not create a temp dir");
+
+	let status = Command::new(cargo_bin("substrate"))
+		.args(&["benchmark", "extrinsic", "--dev"])
+		.arg("-d")
+		.arg(base_dir.path())
+		.args(&["--pallet", pallet, "--extrinsic", extrinsic])
+		// Run with low repeats for faster execution.
+		.args(["--warmup=10", "--repeat=10", "--max-ext-per-block=10"])
+		.status()
+		.unwrap();
+
+	assert!(status.success());
+}

substrate/utils/frame/benchmarking-cli/src/extrinsic/bench.rs

@@ -36,8 +36,8 @@ use log::info;
 use serde::Serialize;
 use std::{marker::PhantomData, sync::Arc, time::Instant};
 
-use super::cmd::ExtrinsicBuilder;
-use crate::shared::Stats;
+use super::ExtrinsicBuilder;
+use crate::shared::{StatSelect, Stats};
 
 /// Parameters to configure an *overhead* benchmark.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
@@ -60,21 +60,11 @@ pub struct BenchmarkParams {
 /// The results of multiple runs in nano seconds.
 pub(crate) type BenchRecord = Vec<u64>;
 
-/// Type of a benchmark.
-#[derive(Serialize, Clone, PartialEq, Copy)]
-pub(crate) enum BenchmarkType {
-	/// Measure the per-extrinsic execution overhead.
-	Extrinsic,
-	/// Measure the per-block execution overhead.
-	Block,
-}
-
 /// Holds all objects needed to run the *overhead* benchmarks.
 pub(crate) struct Benchmark<Block, BA, C> {
 	client: Arc<C>,
 	params: BenchmarkParams,
 	inherent_data: sp_inherents::InherentData,
-	ext_builder: Arc<dyn ExtrinsicBuilder>,
 	_p: PhantomData<(Block, BA)>,
 }
 
@@ -90,23 +80,51 @@ where
 		client: Arc<C>,
 		params: BenchmarkParams,
 		inherent_data: sp_inherents::InherentData,
-		ext_builder: Arc<dyn ExtrinsicBuilder>,
 	) -> Self {
-		Self { client, params, inherent_data, ext_builder, _p: PhantomData }
+		Self { client, params, inherent_data, _p: PhantomData }
 	}
 
-	/// Run the specified benchmark.
-	pub fn bench(&self, bench_type: BenchmarkType) -> Result<Stats> {
-		let (block, num_ext) = self.build_block(bench_type)?;
-		let record = self.measure_block(&block, num_ext, bench_type)?;
+	/// Benchmark a block with only inherents.
+	pub fn bench_block(&self) -> Result<Stats> {
+		let (block, _) = self.build_block(None)?;
+		let record = self.measure_block(&block)?;
 		Stats::new(&record)
 	}
 
-	/// Builds a block for the given benchmark type.
+	/// Benchmark the time of an extrinsic in a full block.
+	///
+	/// First benchmarks an empty block, analogous to `bench_block` and use it as baseline.
+	/// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline
+	/// from the result.
+	/// This is necessary to account for the time the inherents use.
+	pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result<Stats> {
+		let (block, _) = self.build_block(None)?;
+		let base = self.measure_block(&block)?;
+		let base_time = Stats::new(&base)?.select(StatSelect::Average);
+
+		let (block, num_ext) = self.build_block(Some(ext_builder))?;
+		let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?;
+		let mut records = self.measure_block(&block)?;
+
+		for r in &mut records {
+			// Subtract the base time.
+			*r = r.saturating_sub(base_time);
+			// Divide by the number of extrinsics in the block.
+			*r = ((*r as f64) / (num_ext as f64)).ceil() as u64;
+		}
+
+		Stats::new(&records)
+	}
+
+	/// Builds a block with some optional extrinsics.
 	///
 	/// Returns the block and the number of extrinsics in the block
 	/// that are not inherents.
-	fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> {
+	/// Returns a block with only inherents if `ext_builder` is `None`.
+	fn build_block(
+		&self,
+		ext_builder: Option<&dyn ExtrinsicBuilder>,
+	) -> Result<(Block, Option<u64>)> {
 		let mut builder = self.client.new_block(Default::default())?;
 		// Create and insert the inherents.
 		let inherents = builder.create_inherents(self.inherent_data.clone())?;
@@ -114,16 +132,18 @@ where
 			builder.push(inherent)?;
 		}
 
-		// Return early if we just want a block with inherents and no additional extrinsics.
-		if bench_type == BenchmarkType::Block {
-			return Ok((builder.build()?.block, 0))
-		}
+		// Return early if `ext_builder` is `None`.
+		let ext_builder = if let Some(ext_builder) = ext_builder {
+			ext_builder
+		} else {
+			return Ok((builder.build()?.block, None))
+		};
 
 		// Put as many extrinsics into the block as possible and count them.
 		info!("Building block, this takes some time...");
 		let mut num_ext = 0;
 		for nonce in 0..self.max_ext_per_block() {
-			let ext = self.ext_builder.remark(nonce)?;
+			let ext = ext_builder.build(nonce)?;
 			match builder.push(ext.clone()) {
 				Ok(()) => {},
 				Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
@@ -139,20 +159,12 @@ where
 		info!("Extrinsics per block: {}", num_ext);
 		let block = builder.build()?.block;
 
-		Ok((block, num_ext))
+		Ok((block, Some(num_ext)))
 	}
 
 	/// Measures the time that it take to execute a block or an extrinsic.
-	fn measure_block(
-		&self,
-		block: &Block,
-		num_ext: u64,
-		bench_type: BenchmarkType,
-	) -> Result<BenchRecord> {
+	fn measure_block(&self, block: &Block) -> Result<BenchRecord> {
 		let mut record = BenchRecord::new();
-		if bench_type == BenchmarkType::Extrinsic && num_ext == 0 {
-			return Err("Cannot measure the extrinsic time of an empty block".into())
-		}
 		let genesis = BlockId::Number(Zero::zero());
 
 		info!("Running {} warmups...", self.params.warmup);
@@ -176,12 +188,7 @@ where
 				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
 
 			let elapsed = start.elapsed().as_nanos();
-			if bench_type == BenchmarkType::Extrinsic {
-				// Checked for non-zero div above.
-				record.push((elapsed as f64 / num_ext as f64).ceil() as u64);
-			} else {
-				record.push(elapsed as u64);
-			}
+			record.push(elapsed as u64);
 		}
 
 		Ok(record)
@@ -191,21 +198,3 @@ where
 		self.params.max_ext_per_block.unwrap_or(u32::MAX)
 	}
 }
-
-impl BenchmarkType {
-	/// Short name of the benchmark type.
-	pub(crate) fn short_name(&self) -> &'static str {
-		match self {
-			Self::Extrinsic => "extrinsic",
-			Self::Block => "block",
-		}
-	}
-
-	/// Long name of the benchmark type.
-	pub(crate) fn long_name(&self) -> &'static str {
-		match self {
-			Self::Extrinsic => "ExtrinsicBase",
-			Self::Block => "BlockExecution",
-		}
-	}
-}

substrate/utils/frame/benchmarking-cli/src/extrinsic/cmd.rs

@@ -0,0 +1,134 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams};
+use sc_client_api::Backend as ClientBackend;
+use sp_api::{ApiExt, ProvideRuntimeApi};
+use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic};
+
+use clap::{Args, Parser};
+use log::info;
+use serde::Serialize;
+use std::{fmt::Debug, sync::Arc};
+
+use super::{
+	bench::{Benchmark, BenchmarkParams},
+	extrinsic_factory::ExtrinsicFactory,
+};
+
+/// Benchmark the execution time of different extrinsics.
+///
+/// This is calculated by filling a block with a specific extrinsic and executing the block.
+/// The result time is then divided by the number of extrinsics in that block.
+///
+/// NOTE: The BlockExecutionWeight is ignored  in this case since it
+// is very small compared to the total block execution time.
+#[derive(Debug, Parser)]
+pub struct ExtrinsicCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub import_params: ImportParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub params: ExtrinsicParams,
+}
+
+/// The params for the [`ExtrinsicCmd`].
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct ExtrinsicParams {
+	#[clap(flatten)]
+	pub bench: BenchmarkParams,
+
+	/// List all available pallets and extrinsics.
+	///
+	/// The format is CSV with header `pallet, extrinsic`.
+	#[clap(long)]
+	pub list: bool,
+
+	/// Pallet name of the extrinsic to benchmark.
+	#[clap(long, value_name = "PALLET", required_unless_present = "list")]
+	pub pallet: Option<String>,
+
+	/// Extrinsic to benchmark.
+	#[clap(long, value_name = "EXTRINSIC", required_unless_present = "list")]
+	pub extrinsic: Option<String>,
+}
+
+impl ExtrinsicCmd {
+	/// Benchmark the execution time of a specific type of extrinsic.
+	///
+	/// The output will be printed to console.
+	pub fn run<Block, BA, C>(
+		&self,
+		client: Arc<C>,
+		inherent_data: sp_inherents::InherentData,
+		ext_factory: &ExtrinsicFactory,
+	) -> Result<()>
+	where
+		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+		BA: ClientBackend<Block>,
+		C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+		C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+	{
+		// Short circuit if --list was specified.
+		if self.params.list {
+			let list: Vec<String> = ext_factory.0.iter().map(|b| b.name()).collect();
+			info!(
+				"Listing available extrinsics ({}):\npallet, extrinsic\n{}",
+				list.len(),
+				list.join("\n")
+			);
+			return Ok(())
+		}
+
+		let pallet = self.params.pallet.clone().unwrap_or_default();
+		let extrinsic = self.params.extrinsic.clone().unwrap_or_default();
+		let ext_builder = match ext_factory.try_get(&pallet, &extrinsic) {
+			Some(ext_builder) => ext_builder,
+			None =>
+				return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()),
+		};
+
+		let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data);
+		let stats = bench.bench_extrinsic(ext_builder)?;
+		info!(
+			"Executing a {}::{} extrinsic takes[ns]:\n{:?}",
+			ext_builder.pallet(),
+			ext_builder.extrinsic(),
+			stats
+		);
+
+		Ok(())
+	}
+}
+
+// Boilerplate
+impl CliConfiguration for ExtrinsicCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		Some(&self.import_params)
+	}
+}

substrate/utils/frame/benchmarking-cli/src/extrinsic/extrinsic_factory.rs

@@ -0,0 +1,70 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Provides the [`ExtrinsicFactory`] and the [`ExtrinsicBuilder`] types.
+//! Is used by the *overhead* and *extrinsic* benchmarks to build extrinsics.
+
+use sp_runtime::OpaqueExtrinsic;
+
+/// Helper to manage [`ExtrinsicBuilder`] instances.
+#[derive(Default)]
+pub struct ExtrinsicFactory(pub Vec<Box<dyn ExtrinsicBuilder>>);
+
+impl ExtrinsicFactory {
+	/// Returns a builder for a pallet and extrinsic name.
+	///
+	/// Is case in-sensitive.
+	pub fn try_get(&self, pallet: &str, extrinsic: &str) -> Option<&dyn ExtrinsicBuilder> {
+		let pallet = pallet.to_lowercase();
+		let extrinsic = extrinsic.to_lowercase();
+
+		self.0
+			.iter()
+			.find(|b| b.pallet() == pallet && b.extrinsic() == extrinsic)
+			.map(|b| b.as_ref())
+	}
+}
+
+/// Used by the benchmark to build signed extrinsics.
+///
+/// The built extrinsics only need to be valid in the first block
+/// who's parent block is the genesis block.
+/// This assumption simplifies the generation of the extrinsics.
+/// The signer should be one of the pre-funded dev accounts.
+pub trait ExtrinsicBuilder {
+	/// Name of the pallet this builder is for.
+	///
+	/// Should be all lowercase.
+	fn pallet(&self) -> &str;
+
+	/// Name of the extrinsic this builder is for.
+	///
+	/// Should be all lowercase.
+	fn extrinsic(&self) -> &str;
+
+	/// Builds an extrinsic.
+	///
+	/// Will be called multiple times with increasing nonces.
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str>;
+}
+
+impl dyn ExtrinsicBuilder + '_ {
+	/// Name of this builder in CSV format: `pallet, extrinsic`.
+	pub fn name(&self) -> String {
+		format!("{}, {}", self.pallet(), self.extrinsic())
+	}
+}

substrate/utils/frame/benchmarking-cli/src/extrinsic/mod.rs

@@ -0,0 +1,27 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmark the time it takes to execute a specific extrinsic.
+//! This is a generalization of the *overhead* benchmark which can only measure `System::Remark`
+//! extrinsics.
+
+pub mod bench;
+pub mod cmd;
+pub mod extrinsic_factory;
+
+pub use cmd::ExtrinsicCmd;
+pub use extrinsic_factory::{ExtrinsicBuilder, ExtrinsicFactory};

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -18,6 +18,7 @@
 //! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.
 
 mod block;
+mod extrinsic;
 mod machine;
 mod overhead;
 mod pallet;
@@ -25,8 +26,9 @@ mod shared;
 mod storage;
 
 pub use block::BlockCmd;
+pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory};
 pub use machine::{MachineCmd, Requirements, SUBSTRATE_REFERENCE_HARDWARE};
-pub use overhead::{ExtrinsicBuilder, OverheadCmd};
+pub use overhead::OverheadCmd;
 pub use pallet::PalletCmd;
 pub use storage::StorageCmd;
 
@@ -41,8 +43,8 @@ pub enum BenchmarkCmd {
 	Storage(StorageCmd),
 	Overhead(OverheadCmd),
 	Block(BlockCmd),
-	#[clap(hide = true)] // Hidden until fully completed.
 	Machine(MachineCmd),
+	Extrinsic(ExtrinsicCmd),
 }
 
 /// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
@@ -58,6 +60,7 @@ macro_rules! unwrap_cmd {
 			BenchmarkCmd::Overhead($cmd) => $code,
 			BenchmarkCmd::Block($cmd) => $code,
 			BenchmarkCmd::Machine($cmd) => $code,
+			BenchmarkCmd::Extrinsic($cmd) => $code,
 		}
 	}
 }

substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs

@@ -31,10 +31,11 @@ use serde::Serialize;
 use std::{fmt::Debug, sync::Arc};
 
 use crate::{
-	overhead::{
-		bench::{Benchmark, BenchmarkParams, BenchmarkType},
-		template::TemplateData,
+	extrinsic::{
+		bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams},
+		ExtrinsicBuilder,
 	},
+	overhead::template::TemplateData,
 	shared::{HostInfoParams, WeightParams},
 };
 
@@ -63,20 +64,20 @@ pub struct OverheadParams {
 
 	#[allow(missing_docs)]
 	#[clap(flatten)]
-	pub bench: BenchmarkParams,
+	pub bench: ExtrinsicBenchmarkParams,
 
 	#[allow(missing_docs)]
 	#[clap(flatten)]
 	pub hostinfo: HostInfoParams,
 }
 
-/// Used by the benchmark to build signed extrinsics.
-///
-/// The built extrinsics only need to be valid in the first block
-/// who's parent block is the genesis block.
-pub trait ExtrinsicBuilder {
-	/// Build a `System::remark` extrinsic.
-	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str>;
+/// Type of a benchmark.
+#[derive(Serialize, Clone, PartialEq, Copy)]
+pub(crate) enum BenchmarkType {
+	/// Measure the per-extrinsic execution overhead.
+	Extrinsic,
+	/// Measure the per-block execution overhead.
+	Block,
 }
 
 impl OverheadCmd {
@@ -89,7 +90,7 @@ impl OverheadCmd {
 		cfg: Configuration,
 		client: Arc<C>,
 		inherent_data: sp_inherents::InherentData,
-		ext_builder: Arc<dyn ExtrinsicBuilder>,
+		ext_builder: &dyn ExtrinsicBuilder,
 	) -> Result<()>
 	where
 		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
@@ -97,18 +98,21 @@ impl OverheadCmd {
 		C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
 		C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
 	{
-		let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder);
+		if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" {
+			return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into());
+		}
+		let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data);
 
 		// per-block execution overhead
 		{
-			let stats = bench.bench(BenchmarkType::Block)?;
+			let stats = bench.bench_block()?;
 			info!("Per-block execution overhead [ns]:\n{:?}", stats);
 			let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?;
 			template.write(&self.params.weight.weight_path)?;
 		}
 		// per-extrinsic execution overhead
 		{
-			let stats = bench.bench(BenchmarkType::Extrinsic)?;
+			let stats = bench.bench_extrinsic(ext_builder)?;
 			info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats);
 			let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?;
 			template.write(&self.params.weight.weight_path)?;
@@ -118,6 +122,24 @@ impl OverheadCmd {
 	}
 }
 
+impl BenchmarkType {
+	/// Short name of the benchmark type.
+	pub(crate) fn short_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "extrinsic",
+			Self::Block => "block",
+		}
+	}
+
+	/// Long name of the benchmark type.
+	pub(crate) fn long_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "ExtrinsicBase",
+			Self::Block => "BlockExecution",
+		}
+	}
+}
+
 // Boilerplate
 impl CliConfiguration for OverheadCmd {
 	fn shared_params(&self) -> &SharedParams {

substrate/utils/frame/benchmarking-cli/src/overhead/mod.rs

@@ -15,8 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod bench;
 pub mod cmd;
-mod template;
+pub mod template;
 
-pub use cmd::{ExtrinsicBuilder, OverheadCmd};
+pub use cmd::OverheadCmd;

substrate/utils/frame/benchmarking-cli/src/overhead/template.rs

@@ -27,7 +27,7 @@ use serde::Serialize;
 use std::{env, fs, path::PathBuf};
 
 use crate::{
-	overhead::{bench::BenchmarkType, cmd::OverheadParams},
+	overhead::cmd::{BenchmarkType, OverheadParams},
 	shared::{Stats, UnderscoreHelper},
 };