Benchmarks Writer CLI (#6567)

* initial mockup * add and wipe * track writes * start to add to pipeline * return all reads/writes * Log reads and writes from bench db * causes panic * Allow multiple commits * commit before ending benchmark * doesn't work??? * fix * Update lib.rs * switch to struct for `BenchmarkResults` * add to output * fix test * line width * @kianenigma review * Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405) * hardcoded whitelist * Add whitelist to pipeline * Remove whitelist pipeline from CLI, add to runtime * clean-up unused db initialized whitelist * Add regression analysis to DB Tracking (#6475) * Add selector * add tests * debug formatter for easy formula * initial idea * use all benchmarks * broken * working without trait * Make work for multiple pallets * Fix merge issues * writer appends to file * implement () for balances weight trait * update name of trait * Weights to WeightInfo * auto trait writer * Heap pages are configurable * clean out runtime changes * more clean up * Fix string generation * Update comments * Update bin/node/runtime/src/lib.rs Co-authored-by: arkpar <arkady.paronyan@gmail.com>
2026-04-26 15:47:58 +00:00 · 2020-07-06 11:34:24 +02:00
parent a9c21b8b84
commit 2019f70768
8 changed files with 266 additions and 36 deletions
@@ -55,7 +55,7 @@ impl BenchmarkCmd {
 		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
 			wasm_method,
-			None, // heap pages
+			self.heap_pages,
 			2, // The runtime instances cache size.
 		);

@@ -89,6 +89,16 @@ impl BenchmarkCmd {
 		let results = <std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
 			.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;

+		if self.output {
+			if self.weight_trait {
+				let mut file = crate::writer::open_file("traits.rs")?;
+				crate::writer::write_trait(&mut file, results.clone())?;
+			} else {
+				let mut file = crate::writer::open_file("benchmarks.rs")?;
+				crate::writer::write_results(&mut file, results.clone())?;
+			}
+		}
+
 		match results {
 			Ok(batches) => for batch in batches.into_iter() {
 				// Print benchmark metadata
@@ -16,6 +16,7 @@
 // limitations under the License.

 mod command;
+mod writer;

 use sc_cli::{ExecutionStrategy, WasmExecutionMethod};
 use std::fmt::Debug;
@@ -59,6 +60,18 @@ pub struct BenchmarkCmd {
 	#[structopt(long)]
 	pub no_min_squares: bool,

+	/// Output the benchmarks to a Rust file.
+	#[structopt(long)]
+	pub output: bool,
+
+	/// Output the trait definition to a Rust file.
+	#[structopt(long)]
+	pub weight_trait: bool,
+
+	/// Set the heap pages while running benchmarks.
+	#[structopt(long)]
+	pub heap_pages: Option<u64>,
+
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
 	pub shared_params: sc_cli::SharedParams,
@@ -0,0 +1,191 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Outputs benchmark results to Rust files that can be ingested by the runtime.
+
+use std::fs::{File, OpenOptions};
+use std::io::prelude::*;
+use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis};
+use inflector::Inflector;
+
+pub fn open_file(path: &str) -> Result<File, std::io::Error> {
+	OpenOptions::new()
+		.create(true)
+		.write(true)
+		.append(true)
+		.open(path)
+}
+
+pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// trait wrapper
+			write!(file, "// {}\n", pallet_string).unwrap();
+			write!(file, "pub trait WeightInfo {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight;\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	// Reset
+	current_pallet = Vec::<u8>::new();
+
+	batches.iter().for_each(|batch| {
+
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// impl trait
+			write!(file, "\n").unwrap();
+			write!(file, "impl WeightInfo for () {{\n").unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "_{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}
+
+pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
+	let batches = batches.unwrap();
+
+	let mut current_pallet = Vec::<u8>::new();
+
+	// general imports
+	write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap();
+
+	batches.iter().for_each(|batch| {
+
+		let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
+		let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
+
+		// only create new trait definitions when we go to a new pallet
+		if batch.pallet != current_pallet {
+			if !current_pallet.is_empty() {
+				// close trait
+				write!(file, "}}\n").unwrap();
+			}
+
+			// struct for weights
+			write!(file, "pub struct WeightFor{};\n",
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			// trait wrapper
+			write!(file, "impl {}::WeightInfo for WeightFor{} {{\n",
+				pallet_string,
+				pallet_string.to_pascal_case(),
+			).unwrap();
+
+			current_pallet = batch.pallet.clone()
+		}
+
+		// function name
+		write!(file, "	fn {}(", benchmark_string).unwrap();
+
+		// params
+		let components = &batch.results[0].components;
+		for component in components {
+			write!(file, "{:?}: u32, ", component.0).unwrap();
+		}
+		// return value
+		write!(file, ") -> Weight {{\n").unwrap();
+
+		let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap();
+		write!(file, "		({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap();
+		extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
+				slope.saturating_mul(1000),
+				name,
+			).unwrap();
+		});
+
+		let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap();
+		reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap();
+		write!(file, "			.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap();
+		writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| {
+			write!(file, "			.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n",
+				slope,
+				name,
+			).unwrap();
+		});
+
+		// close function
+		write!(file, "	}}\n").unwrap();
+	});
+
+	// final close trait
+	write!(file, "}}\n").unwrap();
+
+	Ok(())
+}