Expose Benchmarking Component Ranges (#11545)

* Add component ranges to benchmarking

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Adding component ranges to templates

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Fix tests

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* cargo run --quiet --profile=production  --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=frame_system --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/system/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* tweak script to reduce diff

* cargo run --quiet --profile=production  --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark pallet --chain=dev --steps=50 --repeat=20 --pallet=pallet_identity --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/identity/src/weights.rs --template=./.maintain/frame-weight-template.hbs

Co-authored-by: Parity Bot <admin@parity.io>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/utils/frame/benchmarking-cli/src/pallet/command.rs

@@ -29,6 +29,7 @@ use sc_cli::{
 use sc_client_db::BenchmarkingState;
 use sc_executor::NativeElseWasmExecutor;
 use sc_service::{Configuration, NativeExecutionDispatch};
+use serde::Serialize;
 use sp_core::offchain::{
 	testing::{TestOffchainExt, TestTransactionPoolExt},
 	OffchainDbExt, OffchainWorkerExt, TransactionPoolExt,
@@ -37,7 +38,18 @@ use sp_externalities::Extensions;
 use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use sp_state_machine::StateMachine;
-use std::{fmt::Debug, fs, sync::Arc, time};
+use std::{collections::HashMap, fmt::Debug, fs, sync::Arc, time};
+
+/// The inclusive range of a component.
+#[derive(Serialize, Debug, Clone, Eq, PartialEq)]
+pub(crate) struct ComponentRange {
+	/// Name of the component.
+	name: String,
+	/// Minimal valid value of the component.
+	min: u32,
+	/// Maximal valid value of the component.
+	max: u32,
+}
 
 // This takes multiple benchmark batches and combines all the results where the pallet, instance,
 // and benchmark are the same.
@@ -212,6 +224,9 @@ impl PalletCmd {
 		let mut batches = Vec::new();
 		let mut batches_db = Vec::new();
 		let mut timer = time::SystemTime::now();
+		// Maps (pallet, extrinsic) to its component ranges.
+		let mut component_ranges = HashMap::<(Vec<u8>, Vec<u8>), Vec<ComponentRange>>::new();
+
 		for (pallet, extrinsic, components) in benchmarks_to_run {
 			let all_components = if components.is_empty() {
 				vec![Default::default()]
@@ -244,6 +259,11 @@ impl PalletCmd {
 							.collect();
 						all_components.push(c);
 					}
+
+					component_ranges
+						.entry((pallet.clone(), extrinsic.clone()))
+						.or_default()
+						.push(ComponentRange { name: name.to_string(), min: lowest, max: highest });
 				}
 				all_components
 			};
@@ -366,7 +386,7 @@ impl PalletCmd {
 
 		// Create the weights.rs file.
 		if let Some(output_path) = &self.output {
-			writer::write_results(&batches, &storage_info, output_path, self)?;
+			writer::write_results(&batches, &storage_info, &component_ranges, output_path, self)?;
 		}
 
 		// Jsonify the result and write it to a file or stdout if desired.

substrate/utils/frame/benchmarking-cli/src/pallet/template.hbs

@@ -25,6 +25,9 @@ impl<T: frame_system::Config> {{pallet}}::WeightInfo for WeightInfo<T> {
 	{{#each benchmark.comments as |comment|}}
 	// {{comment}}
 	{{/each}}
+	{{#each benchmark.component_ranges as |range|}}
+	/// The range of component `{{range.name}}` is `[{{range.min}}, {{range.max}}]`.
+	{{/each}}
 	fn {{benchmark.name~}}
 	(
 		{{~#each benchmark.components as |c| ~}}

substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs

@@ -26,7 +26,7 @@ use std::{
 use inflector::Inflector;
 use serde::Serialize;
 
-use crate::{shared::UnderscoreHelper, PalletCmd};
+use crate::{pallet::command::ComponentRange, shared::UnderscoreHelper, PalletCmd};
 use frame_benchmarking::{
 	Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector,
 	RegressionModel,
@@ -67,6 +67,7 @@ struct BenchmarkData {
 	component_weight: Vec<ComponentSlope>,
 	component_reads: Vec<ComponentSlope>,
 	component_writes: Vec<ComponentSlope>,
+	component_ranges: Vec<ComponentRange>,
 	comments: Vec<String>,
 }
 
@@ -118,6 +119,7 @@ fn io_error(s: &str) -> std::io::Error {
 fn map_results(
 	batches: &[BenchmarkBatchSplitResults],
 	storage_info: &[StorageInfo],
+	component_ranges: &HashMap<(Vec<u8>, Vec<u8>), Vec<ComponentRange>>,
 	analysis_choice: &AnalysisChoice,
 ) -> Result<HashMap<(String, String), Vec<BenchmarkData>>, std::io::Error> {
 	// Skip if batches is empty.
@@ -135,7 +137,8 @@ fn map_results(
 
 		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
 		let instance_string = String::from_utf8(batch.instance.clone()).unwrap();
-		let benchmark_data = get_benchmark_data(batch, storage_info, analysis_choice);
+		let benchmark_data =
+			get_benchmark_data(batch, storage_info, &component_ranges, analysis_choice);
 		let pallet_benchmarks = all_benchmarks.entry((pallet_string, instance_string)).or_default();
 		pallet_benchmarks.push(benchmark_data);
 	}
@@ -155,6 +158,8 @@ fn extract_errors(model: &Option<RegressionModel>) -> impl Iterator<Item = u128>
 fn get_benchmark_data(
 	batch: &BenchmarkBatchSplitResults,
 	storage_info: &[StorageInfo],
+	// Per extrinsic component ranges.
+	component_ranges: &HashMap<(Vec<u8>, Vec<u8>), Vec<ComponentRange>>,
 	analysis_choice: &AnalysisChoice,
 ) -> BenchmarkData {
 	// You can use this to put any additional comments with the benchmarking output.
@@ -238,6 +243,10 @@ fn get_benchmark_data(
 
 	// We add additional comments showing which storage items were touched.
 	add_storage_comments(&mut comments, &batch.db_results, storage_info);
+	let component_ranges = component_ranges
+		.get(&(batch.pallet.clone(), batch.benchmark.clone()))
+		.map(|c| c.clone())
+		.unwrap_or_default();
 
 	BenchmarkData {
 		name: String::from_utf8(batch.benchmark.clone()).unwrap(),
@@ -248,14 +257,16 @@ fn get_benchmark_data(
 		component_weight: used_extrinsic_time,
 		component_reads: used_reads,
 		component_writes: used_writes,
+		component_ranges,
 		comments,
 	}
 }
 
 // Create weight file from benchmark data and Handlebars template.
-pub fn write_results(
+pub(crate) fn write_results(
 	batches: &[BenchmarkBatchSplitResults],
 	storage_info: &[StorageInfo],
+	component_ranges: &HashMap<(Vec<u8>, Vec<u8>), Vec<ComponentRange>>,
 	path: &PathBuf,
 	cmd: &PalletCmd,
 ) -> Result<(), std::io::Error> {
@@ -305,7 +316,7 @@ pub fn write_results(
 	handlebars.register_escape_fn(|s| -> String { s.to_string() });
 
 	// Organize results by pallet into a JSON map
-	let all_results = map_results(batches, storage_info, &analysis_choice)?;
+	let all_results = map_results(batches, storage_info, component_ranges, &analysis_choice)?;
 	for ((pallet, instance), results) in all_results.iter() {
 		let mut file_path = path.clone();
 		// If a user only specified a directory...
@@ -531,6 +542,7 @@ mod test {
 				test_data(b"second", b"first", BenchmarkParameter::c, 3, 4),
 			],
 			&[],
+			&Default::default(),
 			&AnalysisChoice::default(),
 		)
 		.unwrap();