Benchmarks Writer CLI (#6567)

* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* initial idea

* use all benchmarks

* broken

* working without trait

* Make work for multiple pallets

* Fix merge issues

* writer appends to file

* implement () for balances weight trait

* update name of trait

* Weights to WeightInfo

* auto trait writer

* Heap pages are configurable

* clean out runtime changes

* more clean up

* Fix string generation

* Update comments

* Update bin/node/runtime/src/lib.rs

Co-authored-by: arkpar <arkady.paronyan@gmail.com>

substrate/utils/frame/benchmarking-cli/Cargo.toml

@@ -12,6 +12,7 @@ description = "CLI for benchmarking FRAME"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
+Inflector = "0.11.4"
 frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" }
 sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" }
 sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" }

substrate/utils/frame/benchmarking-cli/src/command.rs

@@ -55,7 +55,7 @@ impl BenchmarkCmd {
 		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
 			wasm_method,
-			None, // heap pages
+			self.heap_pages,
 			2, // The runtime instances cache size.
 		);
 
@@ -89,6 +89,16 @@ impl BenchmarkCmd {
 		let results = <std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
 			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
 
+		if self.output {
+			if self.weight_trait {
+				let mut file = crate::writer::open_file("traits.rs")?;
+				crate::writer::write_trait(&mut file, results.clone())?;
+			} else {
+				let mut file = crate::writer::open_file("benchmarks.rs")?;
+				crate::writer::write_results(&mut file, results.clone())?;
+			}
+		}
+
 		match results {
 			Ok(batches) => for batch in batches.into_iter() {
 				// Print benchmark metadata

substrate/utils/frame/benchmarking-cli/src/lib.rs

@@ -16,6 +16,7 @@
 // limitations under the License.
 
 mod command;
+mod writer;
 
 use sc_cli::{ExecutionStrategy, WasmExecutionMethod};
 use std::fmt::Debug;
@@ -59,6 +60,18 @@ pub struct BenchmarkCmd {
 	#[structopt(long)]
 	pub no_min_squares: bool,
 
+	/// Output the benchmarks to a Rust file.
+	#[structopt(long)]
+	pub output: bool,
+
+	/// Output the trait definition to a Rust file.
+	#[structopt(long)]
+	pub weight_trait: bool,
+
+	/// Set the heap pages while running benchmarks.
+	#[structopt(long)]
+	pub heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
 	pub shared_params: sc_cli::SharedParams,

substrate/utils/frame/benchmarking-cli/src/writer.rs

@@ -0,0 +1,191 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Outputs benchmark results to Rust files that can be ingested by the runtime.
+
+use std::fs::{File, OpenOptions};
+use std::io::prelude::*;
+use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis};
+use inflector::Inflector;
+
+pub fn open_file(path: &str) -> Result<File, std::io::Error> {
+	OpenOptions::new()
+		.create(true)
+		.write(true)
+		.append(true)
+		.open(path)
+}
+
+pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// trait wrapper
+			write!(file, "// {}\n", pallet_string).unwrap();
+			write!(file, "pub trait WeightInfo {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight;\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	// Reset
+	current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// impl trait
+			write!(file, "\n").unwrap();
+			write!(file, "impl WeightInfo for () {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "_{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}
+
+pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	// general imports
+	write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// struct for weights
+			write!(file, "pub struct WeightFor{};\n",
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			// trait wrapper
+			write!(file, "impl {}::WeightInfo for WeightFor{} {{\n",
+				pallet_string,
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{\n").unwrap();
+
+		let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap();
+		write!(file, "		({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap();
+		extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
+				slope.saturating_mul(1000),
+				name,
+			).unwrap();
+		});
+
+		let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap();
+		reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap();
+		writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		// close function
+		write!(file, "	}}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}