Add benchmark extrinsic command (#11456)

* Benchmark extrinsic Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Reduce warmup and repeat Running this 1000 times with a full block takes ~33 minutes 🙈. Reducing it to ~3 minutes per default. Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Make ExistentialDeposit public Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add 'bechmark extrinsic' command Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * fmt Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add --list and cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Unrelated Clippy Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Clippy Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix tests and doc Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move implementations up + fmt Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Dont use parameter_types macro Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cache to_lowercase() call The .to_lowercase() on the builder is actually not needes since its already documented to only return lower case. Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Spelling Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Use correct nightly for fmt... Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Rename ED Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix compile Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Subtract block base weight Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Use tmp folder for test This should already be the case since --dev is passed but somehow not... Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
2026-04-27 08:07:58 +00:00 · 2022-07-19 08:10:15 +02:00
parent 3d95c270e0
commit 1843ae83ba
16 changed files with 537 additions and 109 deletions
@@ -0,0 +1,200 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the core benchmarking logic.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{Error, Result};
+use sc_client_api::Backend as ClientBackend;
+use sp_api::{ApiExt, BlockId, Core, ProvideRuntimeApi};
+use sp_blockchain::{
+	ApplyExtrinsicFailed::Validity,
+	Error::{ApplyExtrinsicFailed, RuntimeApiError},
+};
+use sp_runtime::{
+	traits::{Block as BlockT, Zero},
+	transaction_validity::{InvalidTransaction, TransactionValidityError},
+	OpaqueExtrinsic,
+};
+
+use clap::Args;
+use log::info;
+use serde::Serialize;
+use std::{marker::PhantomData, sync::Arc, time::Instant};
+
+use super::ExtrinsicBuilder;
+use crate::shared::{StatSelect, Stats};
+
+/// Parameters to configure an *overhead* benchmark.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct BenchmarkParams {
+	/// Rounds of warmups before measuring.
+	#[clap(long, default_value = "10")]
+	pub warmup: u32,
+
+	/// How many times the benchmark should be repeated.
+	#[clap(long, default_value = "100")]
+	pub repeat: u32,
+
+	/// Maximal number of extrinsics that should be put into a block.
+	///
+	/// Only useful for debugging.
+	#[clap(long)]
+	pub max_ext_per_block: Option<u32>,
+}
+
+/// The results of multiple runs in nano seconds.
+pub(crate) type BenchRecord = Vec<u64>;
+
+/// Holds all objects needed to run the *overhead* benchmarks.
+pub(crate) struct Benchmark<Block, BA, C> {
+	client: Arc<C>,
+	params: BenchmarkParams,
+	inherent_data: sp_inherents::InherentData,
+	_p: PhantomData<(Block, BA)>,
+}
+
+impl<Block, BA, C> Benchmark<Block, BA, C>
+where
+	Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+	BA: ClientBackend<Block>,
+	C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+	C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+{
+	/// Create a new [`Self`] from the arguments.
+	pub fn new(
+		client: Arc<C>,
+		params: BenchmarkParams,
+		inherent_data: sp_inherents::InherentData,
+	) -> Self {
+		Self { client, params, inherent_data, _p: PhantomData }
+	}
+
+	/// Benchmark a block with only inherents.
+	pub fn bench_block(&self) -> Result<Stats> {
+		let (block, _) = self.build_block(None)?;
+		let record = self.measure_block(&block)?;
+		Stats::new(&record)
+	}
+
+	/// Benchmark the time of an extrinsic in a full block.
+	///
+	/// First benchmarks an empty block, analogous to `bench_block` and use it as baseline.
+	/// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline
+	/// from the result.
+	/// This is necessary to account for the time the inherents use.
+	pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result<Stats> {
+		let (block, _) = self.build_block(None)?;
+		let base = self.measure_block(&block)?;
+		let base_time = Stats::new(&base)?.select(StatSelect::Average);
+
+		let (block, num_ext) = self.build_block(Some(ext_builder))?;
+		let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?;
+		let mut records = self.measure_block(&block)?;
+
+		for r in &mut records {
+			// Subtract the base time.
+			*r = r.saturating_sub(base_time);
+			// Divide by the number of extrinsics in the block.
+			*r = ((*r as f64) / (num_ext as f64)).ceil() as u64;
+		}
+
+		Stats::new(&records)
+	}
+
+	/// Builds a block with some optional extrinsics.
+	///
+	/// Returns the block and the number of extrinsics in the block
+	/// that are not inherents.
+	/// Returns a block with only inherents if `ext_builder` is `None`.
+	fn build_block(
+		&self,
+		ext_builder: Option<&dyn ExtrinsicBuilder>,
+	) -> Result<(Block, Option<u64>)> {
+		let mut builder = self.client.new_block(Default::default())?;
+		// Create and insert the inherents.
+		let inherents = builder.create_inherents(self.inherent_data.clone())?;
+		for inherent in inherents {
+			builder.push(inherent)?;
+		}
+
+		// Return early if `ext_builder` is `None`.
+		let ext_builder = if let Some(ext_builder) = ext_builder {
+			ext_builder
+		} else {
+			return Ok((builder.build()?.block, None))
+		};
+
+		// Put as many extrinsics into the block as possible and count them.
+		info!("Building block, this takes some time...");
+		let mut num_ext = 0;
+		for nonce in 0..self.max_ext_per_block() {
+			let ext = ext_builder.build(nonce)?;
+			match builder.push(ext.clone()) {
+				Ok(()) => {},
+				Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
+					InvalidTransaction::ExhaustsResources,
+				)))) => break, // Block is full
+				Err(e) => return Err(Error::Client(e)),
+			}
+			num_ext += 1;
+		}
+		if num_ext == 0 {
+			return Err("A Block must hold at least one extrinsic".into())
+		}
+		info!("Extrinsics per block: {}", num_ext);
+		let block = builder.build()?.block;
+
+		Ok((block, Some(num_ext)))
+	}
+
+	/// Measures the time that it take to execute a block or an extrinsic.
+	fn measure_block(&self, block: &Block) -> Result<BenchRecord> {
+		let mut record = BenchRecord::new();
+		let genesis = BlockId::Number(Zero::zero());
+
+		info!("Running {} warmups...", self.params.warmup);
+		for _ in 0..self.params.warmup {
+			self.client
+				.runtime_api()
+				.execute_block(&genesis, block.clone())
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+		}
+
+		info!("Executing block {} times", self.params.repeat);
+		// Interesting part here:
+		// Execute a block multiple times and record each execution time.
+		for _ in 0..self.params.repeat {
+			let block = block.clone();
+			let runtime_api = self.client.runtime_api();
+			let start = Instant::now();
+
+			runtime_api
+				.execute_block(&genesis, block)
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+
+			let elapsed = start.elapsed().as_nanos();
+			record.push(elapsed as u64);
+		}
+
+		Ok(record)
+	}
+
+	fn max_ext_per_block(&self) -> u32 {
+		self.params.max_ext_per_block.unwrap_or(u32::MAX)
+	}
+}
@@ -0,0 +1,134 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams};
+use sc_client_api::Backend as ClientBackend;
+use sp_api::{ApiExt, ProvideRuntimeApi};
+use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic};
+
+use clap::{Args, Parser};
+use log::info;
+use serde::Serialize;
+use std::{fmt::Debug, sync::Arc};
+
+use super::{
+	bench::{Benchmark, BenchmarkParams},
+	extrinsic_factory::ExtrinsicFactory,
+};
+
+/// Benchmark the execution time of different extrinsics.
+///
+/// This is calculated by filling a block with a specific extrinsic and executing the block.
+/// The result time is then divided by the number of extrinsics in that block.
+///
+/// NOTE: The BlockExecutionWeight is ignored  in this case since it
+// is very small compared to the total block execution time.
+#[derive(Debug, Parser)]
+pub struct ExtrinsicCmd {
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: SharedParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub import_params: ImportParams,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub params: ExtrinsicParams,
+}
+
+/// The params for the [`ExtrinsicCmd`].
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct ExtrinsicParams {
+	#[clap(flatten)]
+	pub bench: BenchmarkParams,
+
+	/// List all available pallets and extrinsics.
+	///
+	/// The format is CSV with header `pallet, extrinsic`.
+	#[clap(long)]
+	pub list: bool,
+
+	/// Pallet name of the extrinsic to benchmark.
+	#[clap(long, value_name = "PALLET", required_unless_present = "list")]
+	pub pallet: Option<String>,
+
+	/// Extrinsic to benchmark.
+	#[clap(long, value_name = "EXTRINSIC", required_unless_present = "list")]
+	pub extrinsic: Option<String>,
+}
+
+impl ExtrinsicCmd {
+	/// Benchmark the execution time of a specific type of extrinsic.
+	///
+	/// The output will be printed to console.
+	pub fn run<Block, BA, C>(
+		&self,
+		client: Arc<C>,
+		inherent_data: sp_inherents::InherentData,
+		ext_factory: &ExtrinsicFactory,
+	) -> Result<()>
+	where
+		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+		BA: ClientBackend<Block>,
+		C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+		C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+	{
+		// Short circuit if --list was specified.
+		if self.params.list {
+			let list: Vec<String> = ext_factory.0.iter().map(|b| b.name()).collect();
+			info!(
+				"Listing available extrinsics ({}):\npallet, extrinsic\n{}",
+				list.len(),
+				list.join("\n")
+			);
+			return Ok(())
+		}
+
+		let pallet = self.params.pallet.clone().unwrap_or_default();
+		let extrinsic = self.params.extrinsic.clone().unwrap_or_default();
+		let ext_builder = match ext_factory.try_get(&pallet, &extrinsic) {
+			Some(ext_builder) => ext_builder,
+			None =>
+				return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()),
+		};
+
+		let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data);
+		let stats = bench.bench_extrinsic(ext_builder)?;
+		info!(
+			"Executing a {}::{} extrinsic takes[ns]:\n{:?}",
+			ext_builder.pallet(),
+			ext_builder.extrinsic(),
+			stats
+		);
+
+		Ok(())
+	}
+}
+
+// Boilerplate
+impl CliConfiguration for ExtrinsicCmd {
+	fn shared_params(&self) -> &SharedParams {
+		&self.shared_params
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		Some(&self.import_params)
+	}
+}
@@ -0,0 +1,70 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Provides the [`ExtrinsicFactory`] and the [`ExtrinsicBuilder`] types.
+//! Is used by the *overhead* and *extrinsic* benchmarks to build extrinsics.
+
+use sp_runtime::OpaqueExtrinsic;
+
+/// Helper to manage [`ExtrinsicBuilder`] instances.
+#[derive(Default)]
+pub struct ExtrinsicFactory(pub Vec<Box<dyn ExtrinsicBuilder>>);
+
+impl ExtrinsicFactory {
+	/// Returns a builder for a pallet and extrinsic name.
+	///
+	/// Is case in-sensitive.
+	pub fn try_get(&self, pallet: &str, extrinsic: &str) -> Option<&dyn ExtrinsicBuilder> {
+		let pallet = pallet.to_lowercase();
+		let extrinsic = extrinsic.to_lowercase();
+
+		self.0
+			.iter()
+			.find(|b| b.pallet() == pallet && b.extrinsic() == extrinsic)
+			.map(|b| b.as_ref())
+	}
+}
+
+/// Used by the benchmark to build signed extrinsics.
+///
+/// The built extrinsics only need to be valid in the first block
+/// who's parent block is the genesis block.
+/// This assumption simplifies the generation of the extrinsics.
+/// The signer should be one of the pre-funded dev accounts.
+pub trait ExtrinsicBuilder {
+	/// Name of the pallet this builder is for.
+	///
+	/// Should be all lowercase.
+	fn pallet(&self) -> &str;
+
+	/// Name of the extrinsic this builder is for.
+	///
+	/// Should be all lowercase.
+	fn extrinsic(&self) -> &str;
+
+	/// Builds an extrinsic.
+	///
+	/// Will be called multiple times with increasing nonces.
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str>;
+}
+
+impl dyn ExtrinsicBuilder + '_ {
+	/// Name of this builder in CSV format: `pallet, extrinsic`.
+	pub fn name(&self) -> String {
+		format!("{}, {}", self.pallet(), self.extrinsic())
+	}
+}
@@ -0,0 +1,27 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Benchmark the time it takes to execute a specific extrinsic.
+//! This is a generalization of the *overhead* benchmark which can only measure `System::Remark`
+//! extrinsics.
+
+pub mod bench;
+pub mod cmd;
+pub mod extrinsic_factory;
+
+pub use cmd::ExtrinsicCmd;
+pub use extrinsic_factory::{ExtrinsicBuilder, ExtrinsicFactory};