Sub-commands for benchmark (#11164)

* Restructure benchmark commands

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add benchmark block test

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fixup imports

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* CI

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Extend error message

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Apply suggestions from code review

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add commands to node-template

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

substrate/utils/frame/benchmarking-cli/src/block/bench.rs

@@ -34,7 +34,7 @@ use serde::Serialize;
 use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
 use thousands::Separable;
 
-use crate::storage::record::{StatSelect, Stats};
+use crate::shared::{StatSelect, Stats};
 
 /// Log target for printing block weight info.
 const LOG_TARGET: &'static str = "benchmark::block::weight";

substrate/utils/frame/benchmarking-cli/src/block/cmd.rs

@@ -29,7 +29,7 @@ use std::{fmt::Debug, sync::Arc};
 
 use super::bench::{Benchmark, BenchmarkParams};
 
-/// Benchmark the execution time historic blocks.
+/// Benchmark the execution time of historic blocks.
 ///
 /// This can be used to verify that blocks do not use more weight than they consumed
 /// in their `WeightInfo`. Example:
@@ -73,7 +73,7 @@ impl BlockCmd {
 	/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
 	///
 	/// Output will be printed to console.
-	pub async fn run<Block, BA, C>(&self, client: Arc<C>) -> Result<()>
+	pub fn run<Block, BA, C>(&self, client: Arc<C>) -> Result<()>
 	where
 		Block: BlockT<Extrinsic = OpaqueExtrinsic>,
 		BA: ClientBackend<Block>,

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -15,144 +15,85 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod block;
-mod command;
-pub mod overhead;
-mod post_processing;
-mod storage;
-mod writer;
+//! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.
 
-use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
-use std::{fmt::Debug, path::PathBuf};
+mod block;
+mod overhead;
+mod pallet;
+mod shared;
+mod storage;
 
 pub use block::BlockCmd;
 pub use overhead::{ExtrinsicBuilder, OverheadCmd};
+pub use pallet::PalletCmd;
 pub use storage::StorageCmd;
 
-// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
-// used like crate names with `_`
-fn parse_pallet_name(pallet: &str) -> String {
-	pallet.replace("-", "_")
+use sc_cli::{CliConfiguration, DatabaseParams, ImportParams, PruningParams, Result, SharedParams};
+
+/// The root `benchmarking` command.
+///
+/// Has no effect itself besides printing a help menu of the sub-commands.
+#[derive(Debug, clap::Subcommand)]
+pub enum BenchmarkCmd {
+	Pallet(PalletCmd),
+	Storage(StorageCmd),
+	Overhead(OverheadCmd),
+	Block(BlockCmd),
 }
 
-/// The `benchmark` command used to benchmark FRAME Pallets.
-#[derive(Debug, clap::Parser)]
-pub struct BenchmarkCmd {
-	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
-	#[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")]
-	pub pallet: Option<String>,
-
-	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
-	#[clap(short, long, required_unless_present = "list")]
-	pub extrinsic: Option<String>,
-
-	/// Select how many samples we should take across the variable components.
-	#[clap(short, long, default_value = "1")]
-	pub steps: u32,
-
-	/// Indicates lowest values for each of the component ranges.
-	#[clap(long = "low", use_value_delimiter = true)]
-	pub lowest_range_values: Vec<u32>,
-
-	/// Indicates highest values for each of the component ranges.
-	#[clap(long = "high", use_value_delimiter = true)]
-	pub highest_range_values: Vec<u32>,
-
-	/// Select how many repetitions of this benchmark should run from within the wasm.
-	#[clap(short, long, default_value = "1")]
-	pub repeat: u32,
-
-	/// Select how many repetitions of this benchmark should run from the client.
-	///
-	/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
-	#[clap(long, default_value = "1")]
-	pub external_repeat: u32,
-
-	/// Print the raw results in JSON format.
-	#[clap(long = "json")]
-	pub json_output: bool,
-
-	/// Write the raw results in JSON format into the given file.
-	#[clap(long, conflicts_with = "json-output")]
-	pub json_file: Option<PathBuf>,
-
-	/// Don't print the median-slopes linear regression analysis.
-	#[clap(long)]
-	pub no_median_slopes: bool,
-
-	/// Don't print the min-squares linear regression analysis.
-	#[clap(long)]
-	pub no_min_squares: bool,
-
-	/// Output the benchmarks to a Rust file at the given path.
-	#[clap(long)]
-	pub output: Option<PathBuf>,
-
-	/// Add a header file to your outputted benchmarks
-	#[clap(long)]
-	pub header: Option<PathBuf>,
-
-	/// Path to Handlebars template file used for outputting benchmark results. (Optional)
-	#[clap(long)]
-	pub template: Option<PathBuf>,
-
-	/// Which analysis function to use when outputting benchmarks:
-	/// * min-squares (default)
-	/// * median-slopes
-	/// * max (max of min squares and median slopes for each value)
-	#[clap(long)]
-	pub output_analysis: Option<String>,
-
-	/// Set the heap pages while running benchmarks. If not set, the default value from the client
-	/// is used.
-	#[clap(long)]
-	pub heap_pages: Option<u64>,
-
-	/// Disable verification logic when running benchmarks.
-	#[clap(long)]
-	pub no_verify: bool,
-
-	/// Display and run extra benchmarks that would otherwise not be needed for weight
-	/// construction.
-	#[clap(long)]
-	pub extra: bool,
-
-	/// Estimate PoV size.
-	#[clap(long)]
-	pub record_proof: bool,
-
-	#[allow(missing_docs)]
-	#[clap(flatten)]
-	pub shared_params: sc_cli::SharedParams,
-
-	/// The execution strategy that should be used for benchmarks
-	#[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)]
-	pub execution: Option<ExecutionStrategy>,
-
-	/// Method for executing Wasm runtime code.
-	#[clap(
-		long = "wasm-execution",
-		value_name = "METHOD",
-		possible_values = WasmExecutionMethod::variants(),
-		ignore_case = true,
-		default_value = DEFAULT_WASM_EXECUTION_METHOD,
-	)]
-	pub wasm_method: WasmExecutionMethod,
-
-	/// Limit the memory the database cache can use.
-	#[clap(long = "db-cache", value_name = "MiB", default_value = "1024")]
-	pub database_cache_size: u32,
-
-	/// List the benchmarks that match your query rather than running them.
-	///
-	/// When nothing is provided, we list all benchmarks.
-	#[clap(long)]
-	pub list: bool,
-
-	/// If enabled, the storage info is not displayed in the output next to the analysis.
-	///
-	/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
-	/// template for that purpose.
-	#[clap(long)]
-	pub no_storage_info: bool,
+/// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
+macro_rules! unwrap_cmd {
+	{
+		$self:expr,
+		$cmd:ident,
+		$code:expr
+	} => {
+		match $self {
+			BenchmarkCmd::Pallet($cmd) => $code,
+			BenchmarkCmd::Storage($cmd) => $code,
+			BenchmarkCmd::Overhead($cmd) => $code,
+			BenchmarkCmd::Block($cmd) => $code,
+		}
+	}
+}
+
+/// Forward the [`CliConfiguration`] trait implementation.
+///
+/// Each time a sub-command exposes a new config option, it must be added here.
+impl CliConfiguration for BenchmarkCmd {
+	fn shared_params(&self) -> &SharedParams {
+		unwrap_cmd! {
+			self, cmd, cmd.shared_params()
+		}
+	}
+
+	fn import_params(&self) -> Option<&ImportParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.import_params()
+		}
+	}
+
+	fn database_params(&self) -> Option<&DatabaseParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.database_params()
+		}
+	}
+
+	fn pruning_params(&self) -> Option<&PruningParams> {
+		unwrap_cmd! {
+			self, cmd, cmd.pruning_params()
+		}
+	}
+
+	fn state_cache_size(&self) -> Result<usize> {
+		unwrap_cmd! {
+			self, cmd, cmd.state_cache_size()
+		}
+	}
+
+	fn chain_id(&self, is_dev: bool) -> Result<String> {
+		unwrap_cmd! {
+			self, cmd, cmd.chain_id(is_dev)
+		}
+	}
 }

substrate/utils/frame/benchmarking-cli/src/overhead/bench.rs

@@ -36,7 +36,8 @@ use log::info;
 use serde::Serialize;
 use std::{marker::PhantomData, sync::Arc, time::Instant};
 
-use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats};
+use super::cmd::ExtrinsicBuilder;
+use crate::shared::Stats;
 
 /// Parameters to configure an *overhead* benchmark.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]

substrate/utils/frame/benchmarking-cli/src/overhead/cmd.rs

@@ -35,10 +35,10 @@ use crate::{
 		bench::{Benchmark, BenchmarkParams, BenchmarkType},
 		template::TemplateData,
 	},
-	post_processing::WeightParams,
+	shared::WeightParams,
 };
 
-/// Benchmarks the per-block and per-extrinsic execution overhead.
+/// Benchmark the execution overhead per-block and per-extrinsic.
 #[derive(Debug, Parser)]
 pub struct OverheadCmd {
 	#[allow(missing_docs)]
@@ -76,11 +76,11 @@ pub trait ExtrinsicBuilder {
 }
 
 impl OverheadCmd {
-	/// Measures the per-block and per-extrinsic execution overhead.
+	/// Measure the per-block and per-extrinsic execution overhead.
 	///
 	/// Writes the results to console and into two instances of the
 	/// `weights.hbs` template, one for each benchmark.
-	pub async fn run<Block, BA, C>(
+	pub fn run<Block, BA, C>(
 		&self,
 		cfg: Configuration,
 		client: Arc<C>,

substrate/utils/frame/benchmarking-cli/src/overhead/template.rs

@@ -28,7 +28,7 @@ use std::{env, fs, path::PathBuf};
 
 use crate::{
 	overhead::{bench::BenchmarkType, cmd::OverheadParams},
-	storage::record::Stats,
+	shared::{Stats, UnderscoreHelper},
 };
 
 static VERSION: &'static str = env!("CARGO_PKG_VERSION");
@@ -85,7 +85,7 @@ impl TemplateData {
 	pub fn write(&self, path: &Option<PathBuf>) -> Result<()> {
 		let mut handlebars = Handlebars::new();
 		// Format large integers with underscores.
-		handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper));
+		handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
 		// Don't HTML escape any characters.
 		handlebars.register_escape_fn(|s| -> String { s.to_string() });
 

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::BenchmarkCmd;
+use super::{writer, PalletCmd};
 use codec::{Decode, Encode};
 use frame_benchmarking::{
 	Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter,
@@ -87,7 +87,13 @@ fn combine_batches(
 		.collect::<Vec<_>>()
 }
 
-impl BenchmarkCmd {
+/// Explains possible reasons why the metadata for the benchmarking could not be found.
+const ERROR_METADATA_NOT_FOUND: &'static str = "Did not find the benchmarking metadata. \
+This could mean that you either did not build the node correctly with the \
+`--features runtime-benchmarks` flag, or the chain spec that you are using was \
+not created by a node that was compiled with the flag";
+
+impl PalletCmd {
 	/// Runs the command and benchmarks the chain.
 	pub fn run<BB, ExecDispatch>(&self, config: Configuration) -> Result<()>
 	where
@@ -165,7 +171,7 @@ impl BenchmarkCmd {
 			sp_core::testing::TaskExecutor::new(),
 		)
 		.execute(strategy.into())
-		.map_err(|e| format!("Error getting benchmark list: {}", e))?;
+		.map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?;
 
 		let (list, storage_info) =
 			<(Vec<BenchmarkList>, Vec<StorageInfo>) as Decode>::decode(&mut &result[..])
@@ -359,7 +365,7 @@ impl BenchmarkCmd {
 
 		// Create the weights.rs file.
 		if let Some(output_path) = &self.output {
-			crate::writer::write_results(&batches, &storage_info, output_path, self)?;
+			writer::write_results(&batches, &storage_info, output_path, self)?;
 		}
 
 		// Jsonify the result and write it to a file or stdout if desired.
@@ -414,11 +420,7 @@ impl BenchmarkCmd {
 
 			if !self.no_storage_info {
 				let mut comments: Vec<String> = Default::default();
-				crate::writer::add_storage_comments(
-					&mut comments,
-					&batch.db_results,
-					&storage_info,
-				);
+				writer::add_storage_comments(&mut comments, &batch.db_results, &storage_info);
 				println!("Raw Storage Info\n========");
 				for comment in comments {
 					println!("{}", comment);
@@ -469,7 +471,7 @@ impl BenchmarkCmd {
 	}
 }
 
-impl CliConfiguration for BenchmarkCmd {
+impl CliConfiguration for PalletCmd {
 	fn shared_params(&self) -> &SharedParams {
 		&self.shared_params
 	}

substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs

@@ -0,0 +1,150 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod command;
+mod writer;
+
+use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
+use std::{fmt::Debug, path::PathBuf};
+
+// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
+// used like crate names with `_`
+fn parse_pallet_name(pallet: &str) -> String {
+	pallet.replace("-", "_")
+}
+
+/// Benchmark the extrinsic weight of FRAME Pallets.
+#[derive(Debug, clap::Parser)]
+pub struct PalletCmd {
+	/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
+	#[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")]
+	pub pallet: Option<String>,
+
+	/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
+	#[clap(short, long, required_unless_present = "list")]
+	pub extrinsic: Option<String>,
+
+	/// Select how many samples we should take across the variable components.
+	#[clap(short, long, default_value = "1")]
+	pub steps: u32,
+
+	/// Indicates lowest values for each of the component ranges.
+	#[clap(long = "low", use_value_delimiter = true)]
+	pub lowest_range_values: Vec<u32>,
+
+	/// Indicates highest values for each of the component ranges.
+	#[clap(long = "high", use_value_delimiter = true)]
+	pub highest_range_values: Vec<u32>,
+
+	/// Select how many repetitions of this benchmark should run from within the wasm.
+	#[clap(short, long, default_value = "1")]
+	pub repeat: u32,
+
+	/// Select how many repetitions of this benchmark should run from the client.
+	///
+	/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
+	#[clap(long, default_value = "1")]
+	pub external_repeat: u32,
+
+	/// Print the raw results in JSON format.
+	#[clap(long = "json")]
+	pub json_output: bool,
+
+	/// Write the raw results in JSON format into the given file.
+	#[clap(long, conflicts_with = "json-output")]
+	pub json_file: Option<PathBuf>,
+
+	/// Don't print the median-slopes linear regression analysis.
+	#[clap(long)]
+	pub no_median_slopes: bool,
+
+	/// Don't print the min-squares linear regression analysis.
+	#[clap(long)]
+	pub no_min_squares: bool,
+
+	/// Output the benchmarks to a Rust file at the given path.
+	#[clap(long)]
+	pub output: Option<PathBuf>,
+
+	/// Add a header file to your outputted benchmarks.
+	#[clap(long)]
+	pub header: Option<PathBuf>,
+
+	/// Path to Handlebars template file used for outputting benchmark results. (Optional)
+	#[clap(long)]
+	pub template: Option<PathBuf>,
+
+	/// Which analysis function to use when outputting benchmarks:
+	/// * min-squares (default)
+	/// * median-slopes
+	/// * max (max of min squares and median slopes for each value)
+	#[clap(long)]
+	pub output_analysis: Option<String>,
+
+	/// Set the heap pages while running benchmarks. If not set, the default value from the client
+	/// is used.
+	#[clap(long)]
+	pub heap_pages: Option<u64>,
+
+	/// Disable verification logic when running benchmarks.
+	#[clap(long)]
+	pub no_verify: bool,
+
+	/// Display and run extra benchmarks that would otherwise not be needed for weight
+	/// construction.
+	#[clap(long)]
+	pub extra: bool,
+
+	/// Estimate PoV size.
+	#[clap(long)]
+	pub record_proof: bool,
+
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub shared_params: sc_cli::SharedParams,
+
+	/// The execution strategy that should be used for benchmarks.
+	#[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)]
+	pub execution: Option<ExecutionStrategy>,
+
+	/// Method for executing Wasm runtime code.
+	#[clap(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = WasmExecutionMethod::variants(),
+		ignore_case = true,
+		default_value = DEFAULT_WASM_EXECUTION_METHOD,
+	)]
+	pub wasm_method: WasmExecutionMethod,
+
+	/// Limit the memory the database cache can use.
+	#[clap(long = "db-cache", value_name = "MiB", default_value = "1024")]
+	pub database_cache_size: u32,
+
+	/// List the benchmarks that match your query rather than running them.
+	///
+	/// When nothing is provided, we list all benchmarks.
+	#[clap(long)]
+	pub list: bool,
+
+	/// If enabled, the storage info is not displayed in the output next to the analysis.
+	///
+	/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
+	/// template for that purpose.
+	#[clap(long)]
+	pub no_storage_info: bool,
+}

substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs

@@ -26,7 +26,7 @@ use std::{
 use inflector::Inflector;
 use serde::Serialize;
 
-use crate::BenchmarkCmd;
+use crate::{shared::UnderscoreHelper, PalletCmd};
 use frame_benchmarking::{
 	Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector,
 	RegressionModel,
@@ -68,7 +68,7 @@ struct BenchmarkData {
 	comments: Vec<String>,
 }
 
-// This forwards some specific metadata from the `BenchmarkCmd`
+// This forwards some specific metadata from the `PalletCmd`
 #[derive(Serialize, Default, Debug, Clone)]
 struct CmdData {
 	steps: u32,
@@ -255,7 +255,7 @@ pub fn write_results(
 	batches: &[BenchmarkBatchSplitResults],
 	storage_info: &[StorageInfo],
 	path: &PathBuf,
-	cmd: &BenchmarkCmd,
+	cmd: &PalletCmd,
 ) -> Result<(), std::io::Error> {
 	// Use custom template if provided.
 	let template: String = match &cmd.template {
@@ -416,44 +416,6 @@ pub(crate) fn add_storage_comments(
 	}
 }
 
-// Add an underscore after every 3rd character, i.e. a separator for large numbers.
-fn underscore<Number>(i: Number) -> String
-where
-	Number: std::string::ToString,
-{
-	let mut s = String::new();
-	let i_str = i.to_string();
-	let a = i_str.chars().rev().enumerate();
-	for (idx, val) in a {
-		if idx != 0 && idx % 3 == 0 {
-			s.insert(0, '_');
-		}
-		s.insert(0, val);
-	}
-	s
-}
-
-// A Handlebars helper to add an underscore after every 3rd character,
-// i.e. a separator for large numbers.
-#[derive(Clone, Copy)]
-pub(crate) struct UnderscoreHelper;
-impl handlebars::HelperDef for UnderscoreHelper {
-	fn call<'reg: 'rc, 'rc>(
-		&self,
-		h: &handlebars::Helper,
-		_: &handlebars::Handlebars,
-		_: &handlebars::Context,
-		_rc: &mut handlebars::RenderContext,
-		out: &mut dyn handlebars::Output,
-	) -> handlebars::HelperResult {
-		use handlebars::JsonRender;
-		let param = h.param(0).unwrap();
-		let underscore_param = underscore(param.value().render());
-		out.write(&underscore_param)?;
-		Ok(())
-	}
-}
-
 // A helper to join a string of vectors.
 #[derive(Clone, Copy)]
 struct JoinHelper;

substrate/utils/frame/benchmarking-cli/src/shared/mod.rs

@@ -0,0 +1,65 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Code that is shared among all benchmarking sub-commands.
+
+pub mod record;
+pub mod stats;
+pub mod weight_params;
+
+pub use record::BenchRecord;
+pub use stats::{StatSelect, Stats};
+pub use weight_params::WeightParams;
+
+/// A Handlebars helper to add an underscore after every 3rd character,
+/// i.e. a separator for large numbers.
+#[derive(Clone, Copy)]
+pub struct UnderscoreHelper;
+
+impl handlebars::HelperDef for UnderscoreHelper {
+	fn call<'reg: 'rc, 'rc>(
+		&self,
+		h: &handlebars::Helper,
+		_: &handlebars::Handlebars,
+		_: &handlebars::Context,
+		_rc: &mut handlebars::RenderContext,
+		out: &mut dyn handlebars::Output,
+	) -> handlebars::HelperResult {
+		use handlebars::JsonRender;
+		let param = h.param(0).unwrap();
+		let underscore_param = underscore(param.value().render());
+		out.write(&underscore_param)?;
+		Ok(())
+	}
+}
+
+/// Add an underscore after every 3rd character, i.e. a separator for large numbers.
+fn underscore<Number>(i: Number) -> String
+where
+	Number: std::string::ToString,
+{
+	let mut s = String::new();
+	let i_str = i.to_string();
+	let a = i_str.chars().rev().enumerate();
+	for (idx, val) in a {
+		if idx != 0 && idx % 3 == 0 {
+			s.insert(0, '_');
+		}
+		s.insert(0, val);
+	}
+	s
+}

substrate/utils/frame/benchmarking-cli/src/shared/record.rs

@@ -0,0 +1,72 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`].
+
+use sc_cli::Result;
+use sc_service::Configuration;
+
+use log::info;
+use serde::Serialize;
+use std::{fs, path::PathBuf, time::Duration};
+
+use super::Stats;
+
+/// Raw output of a Storage benchmark.
+#[derive(Debug, Default, Clone, Serialize)]
+pub struct BenchRecord {
+	/// Multi-Map of value sizes and the time that it took to access them.
+	ns_per_size: Vec<(u64, u64)>,
+}
+
+impl BenchRecord {
+	/// Appends a new record. Uses safe casts.
+	pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
+		let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
+		let ns: u64 = d
+			.as_nanos()
+			.try_into()
+			.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
+		self.ns_per_size.push((size, ns));
+		Ok(())
+	}
+
+	/// Returns the statistics for *time* and *value size*.
+	pub fn calculate_stats(self) -> Result<(Stats, Stats)> {
+		let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
+		let size = Stats::new(&size)?;
+		let time = Stats::new(&time)?;
+		Ok((time, size)) // The swap of time/size here is intentional.
+	}
+
+	/// Unless a path is specified, saves the raw results in a json file in the current directory.
+	/// Prefixes it with the DB name and suffixed with `path_suffix`.
+	pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
+		let mut path = PathBuf::from(out_path);
+		if path.is_dir() || path.as_os_str().is_empty() {
+			path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
+			path.set_extension("json");
+		}
+
+		let json = serde_json::to_string_pretty(&self)
+			.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
+
+		fs::write(&path, json)?;
+		info!("Raw data written to {:?}", fs::canonicalize(&path)?);
+		Ok(())
+	}
+}

substrate/utils/frame/benchmarking-cli/src/shared/stats.rs

@@ -15,46 +15,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Calculates statistics and fills out the `weight.hbs` template.
+//! Handles statistics that were generated from benchmarking results and
+//! that can be used to fill out weight templates.
 
 use sc_cli::Result;
-use sc_service::Configuration;
 
-use log::info;
 use serde::Serialize;
-use std::{fmt, fs, path::PathBuf, result, str::FromStr, time::Duration};
-
-/// Raw output of a Storage benchmark.
-#[derive(Debug, Default, Clone, Serialize)]
-pub(crate) struct BenchRecord {
-	/// Multi-Map of value sizes and the time that it took to access them.
-	ns_per_size: Vec<(u64, u64)>,
-}
+use std::{fmt, result, str::FromStr};
 
 /// Various statistics that help to gauge the quality of the produced weights.
 /// Will be written to the weight file and printed to console.
 #[derive(Serialize, Default, Clone)]
-pub(crate) struct Stats {
+pub struct Stats {
 	/// Sum of all values.
-	pub(crate) sum: u64,
+	pub sum: u64,
 	/// Minimal observed value.
-	pub(crate) min: u64,
+	pub min: u64,
 	/// Maximal observed value.
-	pub(crate) max: u64,
+	pub max: u64,
 
 	/// Average of all values.
-	pub(crate) avg: u64,
+	pub avg: u64,
 	/// Median of all values.
-	pub(crate) median: u64,
+	pub median: u64,
 	/// Standard derivation of all values.
-	pub(crate) stddev: f64,
+	pub stddev: f64,
 
 	/// 99th percentile. At least 99% of all values are below this threshold.
-	pub(crate) p99: u64,
+	pub p99: u64,
 	/// 95th percentile. At least 95% of all values are below this threshold.
-	pub(crate) p95: u64,
+	pub p95: u64,
 	/// 75th percentile. At least 75% of all values are below this threshold.
-	pub(crate) p75: u64,
+	pub p75: u64,
 }
 
 /// Selects a specific field from a [`Stats`] object.
@@ -75,44 +67,6 @@ pub enum StatSelect {
 	P75Percentile,
 }
 
-impl BenchRecord {
-	/// Appends a new record. Uses safe casts.
-	pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
-		let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
-		let ns: u64 = d
-			.as_nanos()
-			.try_into()
-			.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
-		self.ns_per_size.push((size, ns));
-		Ok(())
-	}
-
-	/// Returns the statistics for *time* and *value size*.
-	pub(crate) fn calculate_stats(self) -> Result<(Stats, Stats)> {
-		let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
-		let size = Stats::new(&size)?;
-		let time = Stats::new(&time)?;
-		Ok((time, size)) // The swap of time/size here is intentional.
-	}
-
-	/// Unless a path is specified, saves the raw results in a json file in the current directory.
-	/// Prefixes it with the DB name and suffixed with `path_suffix`.
-	pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
-		let mut path = PathBuf::from(out_path);
-		if path.is_dir() || path.as_os_str().is_empty() {
-			path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
-			path.set_extension("json");
-		}
-
-		let json = serde_json::to_string_pretty(&self)
-			.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
-
-		fs::write(&path, json)?;
-		info!("Raw data written to {:?}", fs::canonicalize(&path)?);
-		Ok(())
-	}
-}
-
 impl Stats {
 	/// Calculates statistics and returns them.
 	pub fn new(xs: &Vec<u64>) -> Result<Self> {
@@ -137,7 +91,7 @@ impl Stats {
 	}
 
 	/// Returns the selected stat.
-	pub(crate) fn select(&self, s: StatSelect) -> u64 {
+	pub fn select(&self, s: StatSelect) -> u64 {
 		match s {
 			StatSelect::Maximum => self.max,
 			StatSelect::Average => self.avg,

substrate/utils/frame/benchmarking-cli/src/shared/weight_params.rs

@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Calculates a weight from the statistics of a benchmark result.
+//! Calculates a weight from the [`super::Stats`] of a benchmark result.
 
 use sc_cli::Result;
 
@@ -23,7 +23,7 @@ use clap::Args;
 use serde::Serialize;
 use std::path::PathBuf;
 
-use crate::storage::record::{StatSelect, Stats};
+use super::{StatSelect, Stats};
 
 /// Configures the weight generation.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
@@ -55,7 +55,7 @@ pub struct WeightParams {
 /// `weight_mul` and adding `weight_add`.
 /// Does not use safe casts and can overflow.
 impl WeightParams {
-	pub(crate) fn calc_weight(&self, stat: &Stats) -> Result<u64> {
+	pub fn calc_weight(&self, stat: &Stats) -> Result<u64> {
 		if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() {
 			return Err("invalid floating number for `weight_mul`".into())
 		}
@@ -68,7 +68,7 @@ impl WeightParams {
 #[cfg(test)]
 mod test_weight_params {
 	use super::WeightParams;
-	use crate::storage::record::{StatSelect, Stats};
+	use crate::shared::{StatSelect, Stats};
 
 	#[test]
 	fn calc_weight_works() {

substrate/utils/frame/benchmarking-cli/src/storage/cmd.rs

@@ -34,8 +34,9 @@ use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};
 
 use super::template::TemplateData;
-use crate::post_processing::WeightParams;
-/// Benchmark the storage of a Substrate node with a live chain snapshot.
+use crate::shared::WeightParams;
+
+/// Benchmark the storage speed of a chain snapshot.
 #[derive(Debug, Parser)]
 pub struct StorageCmd {
 	#[allow(missing_docs)]
@@ -99,7 +100,7 @@ pub struct StorageParams {
 impl StorageCmd {
 	/// Calls into the Read and Write benchmarking functions.
 	/// Processes the output and writes it into files and stdout.
-	pub async fn run<Block, BA, C>(
+	pub fn run<Block, BA, C>(
 		&self,
 		cfg: Configuration,
 		client: Arc<C>,

substrate/utils/frame/benchmarking-cli/src/storage/mod.rs

@@ -17,7 +17,6 @@
 
 pub mod cmd;
 pub mod read;
-pub mod record;
 pub mod template;
 pub mod write;
 

substrate/utils/frame/benchmarking-cli/src/storage/read.rs

@@ -27,7 +27,8 @@ use log::info;
 use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
-use super::{cmd::StorageCmd, record::BenchRecord};
+use super::cmd::StorageCmd;
+use crate::shared::BenchRecord;
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to read a single Storage item.

substrate/utils/frame/benchmarking-cli/src/storage/template.rs

@@ -22,7 +22,8 @@ use log::info;
 use serde::Serialize;
 use std::{env, fs, path::PathBuf};
 
-use super::{cmd::StorageParams, record::Stats};
+use super::cmd::StorageParams;
+use crate::shared::{Stats, UnderscoreHelper};
 
 static VERSION: &'static str = env!("CARGO_PKG_VERSION");
 static TEMPLATE: &str = include_str!("./weights.hbs");
@@ -97,7 +98,7 @@ impl TemplateData {
 	pub fn write(&self, path: &Option<PathBuf>, hbs_template: &Option<PathBuf>) -> Result<()> {
 		let mut handlebars = handlebars::Handlebars::new();
 		// Format large integers with underscore.
-		handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper));
+		handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
 		// Don't HTML escape any characters.
 		handlebars.register_escape_fn(|s| -> String { s.to_string() });
 		// Use custom template if provided.

substrate/utils/frame/benchmarking-cli/src/storage/write.rs

@@ -31,7 +31,8 @@ use log::{info, trace};
 use rand::prelude::*;
 use std::{fmt::Debug, sync::Arc, time::Instant};
 
-use super::{cmd::StorageCmd, record::BenchRecord};
+use super::cmd::StorageCmd;
+use crate::shared::BenchRecord;
 
 impl StorageCmd {
 	/// Benchmarks the time it takes to write a single Storage item.