Benchmarks Writer CLI (#6567)

* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* initial idea

* use all benchmarks

* broken

* working without trait

* Make work for multiple pallets

* Fix merge issues

* writer appends to file

* implement () for balances weight trait

* update name of trait

* Weights to WeightInfo

* auto trait writer

* Heap pages are configurable

* clean out runtime changes

* more clean up

* Fix string generation

* Update comments

* Update bin/node/runtime/src/lib.rs

Co-authored-by: arkpar <arkady.paronyan@gmail.com>

substrate/Cargo.lock

@@ -1450,6 +1450,7 @@ dependencies = [
 name = "frame-benchmarking-cli"
 version = "2.0.0-rc4"
 dependencies = [
+ "Inflector",
  "frame-benchmarking",
  "parity-scale-codec",
  "sc-cli",

substrate/bin/node/runtime/src/lib.rs

@@ -1118,7 +1118,6 @@ impl_runtime_apis! {
 
 			let whitelist: Vec<Vec<u8>> = vec![
 				// Block Number
-				// frame_system::Number::<Runtime>::hashed_key().to_vec(),
 				hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(),
 				// Total Issuance
 				hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(),
@@ -1137,25 +1136,25 @@ impl_runtime_apis! {
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
 
-			add_benchmark!(params, batches, b"babe", Babe);
-			add_benchmark!(params, batches, b"balances", Balances);
-			add_benchmark!(params, batches, b"collective", Council);
-			add_benchmark!(params, batches, b"democracy", Democracy);
-			add_benchmark!(params, batches, b"elections", Elections);
-			add_benchmark!(params, batches, b"identity", Identity);
-			add_benchmark!(params, batches, b"im-online", ImOnline);
-			add_benchmark!(params, batches, b"indices", Indices);
-			add_benchmark!(params, batches, b"multisig", Multisig);
-			add_benchmark!(params, batches, b"offences", OffencesBench::<Runtime>);
-			add_benchmark!(params, batches, b"proxy", Proxy);
-			add_benchmark!(params, batches, b"scheduler", Scheduler);
-			add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
-			add_benchmark!(params, batches, b"staking", Staking);
-			add_benchmark!(params, batches, b"system", SystemBench::<Runtime>);
-			add_benchmark!(params, batches, b"timestamp", Timestamp);
-			add_benchmark!(params, batches, b"treasury", Treasury);
-			add_benchmark!(params, batches, b"utility", Utility);
-			add_benchmark!(params, batches, b"vesting", Vesting);
+			add_benchmark!(params, batches, pallet_babe, Babe);
+			add_benchmark!(params, batches, pallet_balances, Balances);
+			add_benchmark!(params, batches, pallet_collective, Council);
+			add_benchmark!(params, batches, pallet_democracy, Democracy);
+			add_benchmark!(params, batches, pallet_elections_phragmen, Elections);
+			add_benchmark!(params, batches, pallet_identity, Identity);
+			add_benchmark!(params, batches, pallet_im_online, ImOnline);
+			add_benchmark!(params, batches, pallet_indices, Indices);
+			add_benchmark!(params, batches, pallet_multisig, Multisig);
+			add_benchmark!(params, batches, pallet_offences, OffencesBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_proxy, Proxy);
+			add_benchmark!(params, batches, pallet_scheduler, Scheduler);
+			add_benchmark!(params, batches, pallet_session, SessionBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_staking, Staking);
+			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
+			add_benchmark!(params, batches, pallet_timestamp, Timestamp);
+			add_benchmark!(params, batches, pallet_treasury, Treasury);
+			add_benchmark!(params, batches, pallet_utility, Utility);
+			add_benchmark!(params, batches, pallet_vesting, Vesting);
 
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)

substrate/frame/benchmarking/src/analysis.rs

@@ -22,11 +22,11 @@ use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionMode
 use crate::BenchmarkResults;
 
 pub struct Analysis {
-	base: u128,
-	slopes: Vec<u128>,
-	names: Vec<String>,
-	value_dists: Option<Vec<(Vec<u32>, u128, u128)>>,
-	model: Option<RegressionModel>,
+	pub base: u128,
+	pub slopes: Vec<u128>,
+	pub names: Vec<String>,
+	pub value_dists: Option<Vec<(Vec<u32>, u128, u128)>>,
+	pub model: Option<RegressionModel>,
 }
 
 pub enum BenchmarkSelector {

substrate/frame/benchmarking/src/lib.rs

@@ -1158,31 +1158,46 @@ macro_rules! impl_benchmark_test {
 /// First create an object that holds in the input parameters for the benchmark:
 ///
 /// ```ignore
-/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
+/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
 /// ```
 ///
+/// The `whitelist` is a `Vec<Vec<u8>>` of storage keys that you would like to skip for DB tracking. For example:
+///
+/// ```ignore
+/// let whitelist: Vec<Vec<u8>> = vec![
+/// 	// Block Number
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(),
+/// 	// Total Issuance
+/// 	hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(),
+/// 	// Execution Phase
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(),
+/// 	// Event Count
+/// 	hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(),
+/// ];
+///
 /// Then define a mutable local variable to hold your `BenchmarkBatch` object:
 ///
 /// ```ignore
 /// let mut batches = Vec::<BenchmarkBatch>::new();
 /// ````
 ///
-/// Then add the pallets you want to benchmark to this object, including the string
-/// you want to use target a particular pallet:
+/// Then add the pallets you want to benchmark to this object, using their crate name and generated
+/// module struct:
 ///
 /// ```ignore
-/// add_benchmark!(params, batches, b"balances", Balances);
-/// add_benchmark!(params, batches, b"identity", Identity);
-/// add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
+/// add_benchmark!(params, batches, pallet_balances, Balances);
+/// add_benchmark!(params, batches, pallet_session, SessionBench::<Runtime>);
+/// add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
 /// ...
 /// ```
 ///
 /// At the end of `dispatch_benchmark`, you should return this batches object.
 #[macro_export]
 macro_rules! add_benchmark {
-	( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
+	( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => (
+		let name_string = stringify!($name).as_bytes();
 		let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params;
-		if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
+		if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] {
 			if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
 				for benchmark in $( $location )*::benchmarks().into_iter() {
 					$batches.push($crate::BenchmarkBatch {
@@ -1194,7 +1209,7 @@ macro_rules! add_benchmark {
 							repeat,
 							whitelist,
 						)?,
-						pallet: $name.to_vec(),
+						pallet: name_string.to_vec(),
 						benchmark: benchmark.to_vec(),
 					});
 				}
@@ -1208,7 +1223,7 @@ macro_rules! add_benchmark {
 						repeat,
 						whitelist,
 					)?,
-					pallet: $name.to_vec(),
+					pallet: name_string.to_vec(),
 					benchmark: benchmark.clone(),
 				});
 			}

substrate/utils/frame/benchmarking-cli/Cargo.toml

@@ -12,6 +12,7 @@ description = "CLI for benchmarking FRAME"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+Inflector = "0.11.4"
 frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" }
 sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" }
 sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" }

substrate/utils/frame/benchmarking-cli/src/command.rs

@@ -55,7 +55,7 @@ impl BenchmarkCmd {
 		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
 			wasm_method,
-			None, // heap pages
+			self.heap_pages,
 			2, // The runtime instances cache size.
 		);
 
@@ -89,6 +89,16 @@ impl BenchmarkCmd {
 		let results = <std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
 			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
 
+		if self.output {
+			if self.weight_trait {
+				let mut file = crate::writer::open_file("traits.rs")?;
+				crate::writer::write_trait(&mut file, results.clone())?;
+			} else {
+				let mut file = crate::writer::open_file("benchmarks.rs")?;
+				crate::writer::write_results(&mut file, results.clone())?;
+			}
+		}
+
 		match results {
 			Ok(batches) => for batch in batches.into_iter() {
 				// Print benchmark metadata

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -16,6 +16,7 @@
 // limitations under the License.
 
 mod command;
+mod writer;
 
 use sc_cli::{ExecutionStrategy, WasmExecutionMethod};
 use std::fmt::Debug;
@@ -59,6 +60,18 @@ pub struct BenchmarkCmd {
 	#[structopt(long)]
 	pub no_min_squares: bool,
 
+	/// Output the benchmarks to a Rust file.
+	#[structopt(long)]
+	pub output: bool,
+
+	/// Output the trait definition to a Rust file.
+	#[structopt(long)]
+	pub weight_trait: bool,
+
+	/// Set the heap pages while running benchmarks.
+	#[structopt(long)]
+	pub heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
 	pub shared_params: sc_cli::SharedParams,

substrate/utils/frame/benchmarking-cli/src/writer.rs

@@ -0,0 +1,191 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Outputs benchmark results to Rust files that can be ingested by the runtime.
+
+use std::fs::{File, OpenOptions};
+use std::io::prelude::*;
+use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis};
+use inflector::Inflector;
+
+pub fn open_file(path: &str) -> Result<File, std::io::Error> {
+	OpenOptions::new()
+		.create(true)
+		.write(true)
+		.append(true)
+		.open(path)
+}
+
+pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// trait wrapper
+			write!(file, "// {}\n", pallet_string).unwrap();
+			write!(file, "pub trait WeightInfo {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight;\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	// Reset
+	current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// impl trait
+			write!(file, "\n").unwrap();
+			write!(file, "impl WeightInfo for () {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "_{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}
+
+pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	// general imports
+	write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// struct for weights
+			write!(file, "pub struct WeightFor{};\n",
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			// trait wrapper
+			write!(file, "impl {}::WeightInfo for WeightFor{} {{\n",
+				pallet_string,
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{\n").unwrap();
+
+		let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap();
+		write!(file, "		({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap();
+		extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
+				slope.saturating_mul(1000),
+				name,
+			).unwrap();
+		});
+
+		let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap();
+		reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap();
+		writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		// close function
+		write!(file, "	}}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}