Integrate benchmark-overhead command (#5137)

* Integrate benchmark-overhead command

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Beauty fix test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Spellcheck on

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove constants:: module

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update block+extrinsic weights

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Revert "Update block+extrinsic weights"

This reverts commit d77bf977f6bca0b1ded35daa2fd54b97e18a67db.

* Revert "Remove constants:: module"

This reverts commit 2d3bcd0212c438f6b96755900a01762592d90265.

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: parity-processbot <>

polkadot/Cargo.lock

@@ -6459,12 +6459,18 @@ version = "0.9.18"
 dependencies = [
  "beefy-primitives",
  "frame-benchmarking",
+ "frame-benchmarking-cli",
+ "frame-system",
  "frame-system-rpc-runtime-api",
  "kusama-runtime",
  "pallet-mmr-primitives",
+ "pallet-transaction-payment",
  "pallet-transaction-payment-rpc-runtime-api",
+ "polkadot-core-primitives",
+ "polkadot-node-core-parachains-inherent",
  "polkadot-primitives",
  "polkadot-runtime",
+ "polkadot-runtime-common",
  "rococo-runtime",
  "sc-client-api",
  "sc-consensus",
@@ -6476,11 +6482,15 @@ dependencies = [
  "sp-blockchain",
  "sp-consensus",
  "sp-consensus-babe",
+ "sp-core",
  "sp-finality-grandpa",
+ "sp-inherents",
+ "sp-keyring",
  "sp-offchain",
  "sp-runtime",
  "sp-session",
  "sp-storage",
+ "sp-timestamp",
  "sp-transaction-pool",
  "westend-runtime",
 ]

polkadot/cli/src/cli.rs

@@ -61,6 +61,13 @@ pub enum Subcommand {
 	)]
 	BenchmarkBlock(frame_benchmarking_cli::BlockCmd),
 
+	/// Sub command for benchmarking the per-block and per-extrinsic execution overhead.
+	#[clap(
+		name = "benchmark-overhead",
+		about = "Benchmark the per-block and per-extrinsic execution overhead."
+	)]
+	BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd),
+
 	/// Sub command for benchmarking the storage speed.
 	#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
 	BenchmarkStorage(frame_benchmarking_cli::StorageCmd),

polkadot/cli/src/command.rs

@@ -18,11 +18,11 @@ use crate::cli::{Cli, Subcommand};
 use futures::future::TryFutureExt;
 use log::info;
 use sc_cli::{Role, RuntimeVersion, SubstrateCli};
-use service::{self, IdentifyVariant};
+use service::{self, HeaderBackend, IdentifyVariant};
 use sp_core::crypto::Ss58AddressFormatRegistry;
 use std::net::ToSocketAddrs;
 
-pub use crate::error::Error;
+pub use crate::{error::Error, service::BlockId};
 pub use polkadot_performance_test::PerfCheckError;
 
 impl std::convert::From<String> for Error {
@@ -534,6 +534,101 @@ pub fn run() -> Result<()> {
 			#[cfg(not(feature = "polkadot-native"))]
 			unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
 		},
+		Some(Subcommand::BenchmarkOverhead(cmd)) => {
+			use polkadot_client::benchmark_inherent_data;
+
+			let runner = cli.create_runner(cmd)?;
+			let chain_spec = &runner.config().chain_spec;
+			set_default_ss58_version(chain_spec);
+			ensure_dev(chain_spec).map_err(Error::Other)?;
+
+			#[cfg(feature = "rococo-native")]
+			if chain_spec.is_rococo() || chain_spec.is_wococo() || chain_spec.is_versi() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
+					let inherent_data = benchmark_inherent_data(header)
+						.map_err(|e| format!("generating inherent data: {:?}", e))?;
+
+					if let polkadot_client::Client::Rococo(pd) = &*client {
+						Ok((
+							cmd.run(config, pd.clone(), inherent_data, client)
+								.map_err(Error::SubstrateCli),
+							task_manager,
+						))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "kusama-native")]
+			if chain_spec.is_kusama() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
+					let inherent_data = benchmark_inherent_data(header)
+						.map_err(|e| format!("generating inherent data: {:?}", e))?;
+
+					if let polkadot_client::Client::Kusama(pd) = &*client {
+						Ok((
+							cmd.run(config, pd.clone(), inherent_data, client)
+								.map_err(Error::SubstrateCli),
+							task_manager,
+						))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "westend-native")]
+			if chain_spec.is_westend() {
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
+					let inherent_data = benchmark_inherent_data(header)
+						.map_err(|e| format!("generating inherent data: {:?}", e))?;
+
+					if let polkadot_client::Client::Westend(pd) = &*client {
+						Ok((
+							cmd.run(config, pd.clone(), inherent_data, client)
+								.map_err(Error::SubstrateCli),
+							task_manager,
+						))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(feature = "polkadot-native")]
+			{
+				return Ok(runner.async_run(|mut config| {
+					let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;
+
+					let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
+					let inherent_data = benchmark_inherent_data(header)
+						.map_err(|e| format!("generating inherent data: {:?}", e))?;
+
+					if let polkadot_client::Client::Polkadot(pd) = &*client {
+						Ok((
+							cmd.run(config, pd.clone(), inherent_data, client)
+								.map_err(Error::SubstrateCli),
+							task_manager,
+						))
+					} else {
+						unreachable!("Checked above; qed")
+					}
+				})?)
+			}
+
+			#[cfg(not(feature = "polkadot-native"))]
+			unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
+		},
 		Some(Subcommand::BenchmarkStorage(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			let chain_spec = &runner.config().chain_spec;

polkadot/node/client/Cargo.toml

@@ -6,7 +6,10 @@ edition = "2021"
 
 [dependencies]
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master" }
+frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
+pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", branch = "master" }
 pallet-transaction-payment-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+frame-system = { git = "https://github.com/paritytech/substrate", branch = "master" }
 frame-system-rpc-runtime-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -14,6 +17,10 @@ sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-timestamp = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-session = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-finality-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master" }
@@ -37,7 +44,10 @@ kusama-runtime = { path = "../../runtime/kusama", optional = true }
 westend-runtime = { path = "../../runtime/westend", optional = true }
 rococo-runtime = { path = "../../runtime/rococo", optional = true }
 
+polkadot-core-primitives = { path = "../../core-primitives" }
 polkadot-primitives = { path = "../../primitives" }
+polkadot-node-core-parachains-inherent = { path = "../core/parachains-inherent" }
+polkadot-runtime-common = { path = "../../runtime/common" }
 
 [features]
 default = ["polkadot"]

polkadot/node/client/src/lib.rs

@@ -24,13 +24,15 @@ use polkadot_primitives::v2::{
 };
 use sc_client_api::{AuxStore, Backend as BackendT, BlockchainEvents, KeyIterator, UsageProvider};
 use sc_executor::NativeElseWasmExecutor;
-use sp_api::{CallApiAt, NumberFor, ProvideRuntimeApi};
+use sp_api::{CallApiAt, Encode, NumberFor, ProvideRuntimeApi};
 use sp_blockchain::HeaderBackend;
 use sp_consensus::BlockStatus;
+use sp_core::Pair;
+use sp_keyring::Sr25519Keyring;
 use sp_runtime::{
 	generic::{BlockId, SignedBlock},
 	traits::{BlakeTwo256, Block as BlockT},
-	Justifications,
+	Justifications, OpaqueExtrinsic,
 };
 use sp_storage::{ChildInfo, StorageData, StorageKey};
 use std::sync::Arc;
@@ -567,3 +569,222 @@ impl sp_blockchain::HeaderBackend<Block> for Client {
 		}
 	}
 }
+
+/// Provides a `SignedPayload` for any runtime.
+///
+/// Should only be used for benchmarking as it is not tested for regular usage.
+///
+/// The first code block should set up all variables that are needed to create the
+/// `SignedPayload`. The second block can make use of the `SignedPayload`.
+///
+/// This is not done as a trait function since the return type depends on the runtime.
+/// This macro therefore uses the same approach as [`with_client!`].
+macro_rules! with_signed_payload {
+	{
+		$self:ident,
+		{
+			$extra:ident,
+			$client:ident,
+			$raw_payload:ident
+		},
+		{
+			$( $setup:tt )*
+		},
+		(
+			$period:expr,
+			$current_block:expr,
+			$nonce:expr,
+			$tip:expr,
+			$call:expr,
+			$genesis:expr
+		),
+		{
+			$( $usage:tt )*
+		}
+	} => {
+		match $self {
+			#[cfg(feature = "polkadot")]
+			Self::Polkadot($client) => {
+				use polkadot_runtime as runtime;
+
+				$( $setup )*
+
+				let $extra: runtime::SignedExtra = (
+					frame_system::CheckNonZeroSender::<runtime::Runtime>::new(),
+					frame_system::CheckSpecVersion::<runtime::Runtime>::new(),
+					frame_system::CheckTxVersion::<runtime::Runtime>::new(),
+					frame_system::CheckGenesis::<runtime::Runtime>::new(),
+					frame_system::CheckMortality::<runtime::Runtime>::from(sp_runtime::generic::Era::mortal(
+						$period,
+						$current_block,
+					)),
+					frame_system::CheckNonce::<runtime::Runtime>::from($nonce),
+					frame_system::CheckWeight::<runtime::Runtime>::new(),
+					pallet_transaction_payment::ChargeTransactionPayment::<runtime::Runtime>::from($tip),
+					polkadot_runtime_common::claims::PrevalidateAttests::<runtime::Runtime>::new(),
+				);
+
+				let $raw_payload = runtime::SignedPayload::from_raw(
+					$call.clone(),
+					$extra.clone(),
+					(
+						(),
+						runtime::VERSION.spec_version,
+						runtime::VERSION.transaction_version,
+						$genesis.clone(),
+						$genesis,
+						(),
+						(),
+						(),
+						(),
+					),
+				);
+
+				$( $usage )*
+			},
+			#[cfg(feature = "westend")]
+			Self::Westend($client) => {
+				use westend_runtime as runtime;
+
+				$( $setup )*
+
+				signed_payload!($extra, $raw_payload,
+					($period, $current_block, $nonce, $tip, $call, $genesis));
+
+				$( $usage )*
+			},
+			#[cfg(feature = "kusama")]
+			Self::Kusama($client) => {
+				use kusama_runtime as runtime;
+
+				$( $setup )*
+
+				signed_payload!($extra, $raw_payload,
+					($period, $current_block, $nonce, $tip, $call, $genesis));
+
+				$( $usage )*
+			},
+			#[cfg(feature = "rococo")]
+			Self::Rococo($client) => {
+				use rococo_runtime as runtime;
+
+				$( $setup )*
+
+				signed_payload!($extra, $raw_payload,
+					($period, $current_block, $nonce, $tip, $call, $genesis));
+
+				$( $usage )*
+			},
+		}
+	}
+}
+
+/// Generates a `SignedPayload` for the Kusama, Westend and Rococo runtime.
+///
+/// Should only be used for benchmarking as it is not tested for regular usage.
+#[allow(unused_macros)]
+macro_rules! signed_payload {
+	(
+	$extra:ident, $raw_payload:ident,
+	(
+		$period:expr,
+		$current_block:expr,
+		$nonce:expr,
+		$tip:expr,
+		$call:expr,
+		$genesis:expr
+	)
+	) => {
+		let $extra: runtime::SignedExtra = (
+			frame_system::CheckNonZeroSender::<runtime::Runtime>::new(),
+			frame_system::CheckSpecVersion::<runtime::Runtime>::new(),
+			frame_system::CheckTxVersion::<runtime::Runtime>::new(),
+			frame_system::CheckGenesis::<runtime::Runtime>::new(),
+			frame_system::CheckMortality::<runtime::Runtime>::from(
+				sp_runtime::generic::Era::mortal($period, $current_block),
+			),
+			frame_system::CheckNonce::<runtime::Runtime>::from($nonce),
+			frame_system::CheckWeight::<runtime::Runtime>::new(),
+			pallet_transaction_payment::ChargeTransactionPayment::<runtime::Runtime>::from($tip),
+		);
+
+		let $raw_payload = runtime::SignedPayload::from_raw(
+			$call.clone(),
+			$extra.clone(),
+			(
+				(),
+				runtime::VERSION.spec_version,
+				runtime::VERSION.transaction_version,
+				$genesis.clone(),
+				$genesis,
+				(),
+				(),
+				(),
+			),
+		);
+	};
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for Client {
+	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		with_signed_payload! {
+			self,
+			{extra, client, raw_payload},
+			{
+				// First the setup code to init all the variables that are needed
+				// to build the signed extras.
+				use runtime::{Call, SystemCall};
+
+				let call = Call::System(SystemCall::remark { remark: vec![] });
+				let bob = Sr25519Keyring::Bob.pair();
+
+				let period = polkadot_runtime_common::BlockHashCount::get().checked_next_power_of_two().map(|c| c / 2).unwrap_or(2) as u64;
+
+				let current_block = 0;
+				let tip = 0;
+				let genesis = client.usage_info().chain.best_hash;
+			},
+			(period, current_block, nonce, tip, call, genesis),
+			/* The SignedPayload is generated here */
+			{
+				// Use the payload to generate a signature.
+				let signature = raw_payload.using_encoded(|payload| bob.sign(payload));
+
+				let ext = runtime::UncheckedExtrinsic::new_signed(
+					call,
+					sp_runtime::AccountId32::from(bob.public()).into(),
+					polkadot_core_primitives::Signature::Sr25519(signature.clone()),
+					extra,
+				);
+				Ok(ext.into())
+			}
+		}
+	}
+}
+
+/// Generates inherent data for benchmarking Polkadot, Kusama, Westend and Rococo.
+///
+/// Not to be used outside of benchmarking since it returns mocked values.
+pub fn benchmark_inherent_data(
+	header: polkadot_core_primitives::Header,
+) -> std::result::Result<sp_inherents::InherentData, sp_inherents::Error> {
+	use sp_inherents::InherentDataProvider;
+	let mut inherent_data = sp_inherents::InherentData::new();
+
+	// Assume that all runtimes have the `timestamp` pallet.
+	let d = std::time::Duration::from_millis(0);
+	let timestamp = sp_timestamp::InherentDataProvider::new(d.into());
+	timestamp.provide_inherent_data(&mut inherent_data)?;
+
+	let para_data = polkadot_primitives::v2::InherentData {
+		bitfields: Vec::new(),
+		backed_candidates: Vec::new(),
+		disputes: Vec::new(),
+		parent_header: header,
+	};
+
+	polkadot_node_core_parachains_inherent::ParachainsInherentDataProvider::from_data(para_data)
+		.provide_inherent_data(&mut inherent_data)?;
+
+	Ok(inherent_data)
+}

polkadot/node/core/parachains-inherent/src/lib.rs

@@ -42,6 +42,11 @@ pub struct ParachainsInherentDataProvider {
 }
 
 impl ParachainsInherentDataProvider {
+	/// Create a [`Self`] directly from some [`ParachainsInherentData`].
+	pub fn from_data(inherent_data: ParachainsInherentData) -> Self {
+		Self { inherent_data }
+	}
+
 	/// Create a new instance of the [`ParachainsInherentDataProvider`].
 	pub async fn create<C: HeaderBackend<Block>>(
 		client: &C,

polkadot/runtime/kusama/src/lib.rs

@@ -79,6 +79,7 @@ use sp_version::NativeVersion;
 use sp_version::RuntimeVersion;
 use static_assertions::const_assert;
 
+pub use frame_system::Call as SystemCall;
 pub use pallet_balances::Call as BalancesCall;
 pub use pallet_election_provider_multi_phase::Call as EPMCall;
 #[cfg(feature = "std")]

polkadot/runtime/polkadot/src/lib.rs

@@ -81,6 +81,7 @@ use sp_version::NativeVersion;
 use sp_version::RuntimeVersion;
 use static_assertions::const_assert;
 
+pub use frame_system::Call as SystemCall;
 pub use pallet_balances::Call as BalancesCall;
 pub use pallet_election_provider_multi_phase::Call as EPMCall;
 #[cfg(feature = "std")]

polkadot/runtime/rococo/src/lib.rs

@@ -78,6 +78,8 @@ use bridge_runtime_common::messages::{
 	source::estimate_message_dispatch_and_delivery_fee, MessageBridge,
 };
 
+pub use frame_system::Call as SystemCall;
+
 /// Constant values used within the runtime.
 use rococo_runtime_constants::{currency::*, fee::*, time::*};
 

polkadot/runtime/westend/src/lib.rs

@@ -74,6 +74,7 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*};
 use sp_version::NativeVersion;
 use sp_version::RuntimeVersion;
 
+pub use frame_system::Call as SystemCall;
 pub use pallet_election_provider_multi_phase::Call as EPMCall;
 #[cfg(feature = "std")]
 pub use pallet_staking::StakerStatus;

polkadot/tests/benchmark_overhead_works.rs

@@ -0,0 +1,69 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::{process::Command, result::Result};
+use tempfile::tempdir;
+
+static RUNTIMES: [&'static str; 6] = ["polkadot", "kusama", "westend", "rococo", "wococo", "versi"];
+
+/// `benchmark-overhead` works for all dev runtimes.
+#[test]
+fn benchmark_overhead_works() {
+	for runtime in RUNTIMES {
+		let runtime = format!("{}-dev", runtime);
+		assert!(benchmark_overhead(runtime).is_ok());
+	}
+}
+
+/// `benchmark-overhead` rejects all non-dev runtimes.
+#[test]
+fn benchmark_overhead_rejects_non_dev_runtimes() {
+	for runtime in RUNTIMES {
+		assert!(benchmark_overhead(runtime.into()).is_err());
+	}
+}
+
+fn benchmark_overhead(runtime: String) -> Result<(), String> {
+	let tmp_dir = tempdir().expect("could not create a temp dir");
+	let base_path = tmp_dir.path();
+
+	// Invoke `benchmark-overhead` with all options to make sure that they are valid.
+	let status = Command::new(cargo_bin("polkadot"))
+		.args(["benchmark-overhead", "--chain", &runtime])
+		.arg("-d")
+		.arg(base_path)
+		.arg("--weight-path")
+		.arg(base_path)
+		.args(["--warmup", "5", "--repeat", "5"])
+		.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
+		// Only put 5 extrinsics into the block otherwise it takes forever to build it
+		// especially for a non-release builds.
+		.args(["--max-ext-per-block", "5"])
+		.status()
+		.map_err(|e| format!("command failed: {:?}", e))?;
+
+	if !status.success() {
+		return Err("Command failed".into())
+	}
+
+	// Weight files have been created.
+	assert!(base_path.join("block_weights.rs").exists());
+	assert!(base_path.join("extrinsic_weights.rs").exists());
+	Ok(())
+}