Make Benchmark Output Analysis Function Configurable (#8228)

* Integrate `output-analysis`

* fix test

* use default

* cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_system --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/system/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* Update frame/system/src/weights.rs

* cargo run --release --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_system --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/system/src/weights.rs --template=./.maintain/frame-weight-template.hbs --output-analysis=max

* Update frame/system/src/weights.rs

* dont discard value_dist and model

* feedback

Co-authored-by: Parity Benchmarking Bot <admin@parity.io>

substrate/frame/benchmarking/src/analysis.rs

@@ -18,6 +18,7 @@
 //! Tools for analyzing the benchmark results.
 
 use std::collections::BTreeMap;
+use core::convert::TryFrom;
 use linregress::{FormulaRegressionBuilder, RegressionDataBuilder};
 use crate::BenchmarkResults;
 
@@ -31,6 +32,7 @@ pub struct Analysis {
 	pub model: Option<RegressionModel>,
 }
 
+#[derive(Clone, Copy)]
 pub enum BenchmarkSelector {
 	ExtrinsicTime,
 	StorageRootTime,
@@ -38,6 +40,40 @@ pub enum BenchmarkSelector {
 	Writes,
 }
 
+#[derive(Debug)]
+pub enum AnalysisChoice {
+	/// Use minimum squares regression for analyzing the benchmarking results.
+	MinSquares,
+	/// Use median slopes for analyzing the benchmarking results.
+	MedianSlopes,
+	/// Use the maximum values among all other analysis functions for the benchmarking results.
+	Max,
+}
+
+impl Default for AnalysisChoice {
+	fn default() -> Self {
+		AnalysisChoice::MinSquares
+	}
+}
+
+impl TryFrom<Option<String>> for AnalysisChoice {
+	type Error = &'static str;
+
+	fn try_from(s: Option<String>) -> Result<Self, Self::Error> {
+		match s {
+			None => Ok(AnalysisChoice::default()),
+			Some(i) => {
+				match &i[..] {
+					"min-squares" | "min_squares" => Ok(AnalysisChoice::MinSquares),
+					"median-slopes" | "median_slopes" => Ok(AnalysisChoice::MedianSlopes),
+					"max" => Ok(AnalysisChoice::Max),
+					_ => Err("invalid analysis string")
+				}
+			}
+		}
+	}
+}
+
 impl Analysis {
 	// Useful for when there are no components, and we just need an median value of the benchmark results.
 	// Note: We choose the median value because it is more robust to outliers.
@@ -215,6 +251,39 @@ impl Analysis {
 			model: Some(model),
 		})
 	}
+
+	pub fn max(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
+		let median_slopes = Self::median_slopes(r, selector);
+		let min_squares = Self::min_squares_iqr(r, selector);
+
+		if median_slopes.is_none() || min_squares.is_none() {
+			return None;
+		}
+
+		let median_slopes = median_slopes.unwrap();
+		let min_squares = min_squares.unwrap();
+
+		let base = median_slopes.base.max(min_squares.base);
+		let slopes = median_slopes.slopes.into_iter()
+			.zip(min_squares.slopes.into_iter())
+			.map(|(a, b): (u128, u128)| { a.max(b) })
+			.collect::<Vec<u128>>();
+		// components should always be in the same order
+		median_slopes.names.iter()
+			.zip(min_squares.names.iter())
+			.for_each(|(a, b)| assert!(a == b, "benchmark results not in the same order"));
+		let names = median_slopes.names;
+		let value_dists = min_squares.value_dists;
+		let model = min_squares.model;
+
+		Some(Self {
+			base,
+			slopes,
+			names,
+			value_dists,
+			model,
+		})
+	}
 }
 
 fn ms(mut nanos: u128) -> String {

substrate/frame/benchmarking/src/lib.rs

@@ -26,7 +26,7 @@ mod analysis;
 
 pub use utils::*;
 #[cfg(feature = "std")]
-pub use analysis::{Analysis, BenchmarkSelector, RegressionModel};
+pub use analysis::{Analysis, BenchmarkSelector, RegressionModel, AnalysisChoice};
 #[doc(hidden)]
 pub use sp_io::storage::root as storage_root;
 #[doc(hidden)]

substrate/frame/system/src/weights.rs

@@ -18,7 +18,7 @@
 //! Autogenerated weights for frame_system
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0
-//! DATE: 2021-02-27, STEPS: [50, ], REPEAT: 20, LOW RANGE: [], HIGH RANGE: []
+//! DATE: 2021-02-28, STEPS: [50, ], REPEAT: 20, LOW RANGE: [], HIGH RANGE: []
 //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128
 
 // Executed Command:
@@ -34,6 +34,7 @@
 // --heap-pages=4096
 // --output=./frame/system/src/weights.rs
 // --template=./.maintain/frame-weight-template.hbs
+// --output-analysis=max
 
 
 #![allow(unused_parens)]
@@ -57,38 +58,38 @@ pub trait WeightInfo {
 pub struct SubstrateWeight<T>(PhantomData<T>);
 impl<T: crate::Config> WeightInfo for SubstrateWeight<T> {
 	fn remark(_b: u32, ) -> Weight {
-		(1_296_000 as Weight)
+		(1_345_000 as Weight)
 	}
 	fn remark_with_event(b: u32, ) -> Weight {
-		(13_474_000 as Weight)
+		(9_697_000 as Weight)
 			// Standard Error: 0
 			.saturating_add((1_000 as Weight).saturating_mul(b as Weight))
 	}
 	fn set_heap_pages() -> Weight {
-		(2_024_000 as Weight)
+		(2_070_000 as Weight)
 			.saturating_add(T::DbWeight::get().writes(1 as Weight))
 	}
 	fn set_changes_trie_config() -> Weight {
-		(10_551_000 as Weight)
+		(10_111_000 as Weight)
 			.saturating_add(T::DbWeight::get().reads(1 as Weight))
 			.saturating_add(T::DbWeight::get().writes(2 as Weight))
 	}
 	fn set_storage(i: u32, ) -> Weight {
 		(0 as Weight)
 			// Standard Error: 0
-			.saturating_add((612_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add((619_000 as Weight).saturating_mul(i as Weight))
 			.saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
 	}
 	fn kill_storage(i: u32, ) -> Weight {
-		(562_000 as Weight)
+		(1_647_000 as Weight)
 			// Standard Error: 0
-			.saturating_add((442_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add((460_000 as Weight).saturating_mul(i as Weight))
 			.saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
 	}
 	fn kill_prefix(p: u32, ) -> Weight {
-		(10_499_000 as Weight)
-			// Standard Error: 1_000
-			.saturating_add((840_000 as Weight).saturating_mul(p as Weight))
+		(10_678_000 as Weight)
+			// Standard Error: 0
+			.saturating_add((862_000 as Weight).saturating_mul(p as Weight))
 			.saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
 	}
 }
@@ -96,38 +97,38 @@ impl<T: crate::Config> WeightInfo for SubstrateWeight<T> {
 // For backwards compatibility and tests
 impl WeightInfo for () {
 	fn remark(_b: u32, ) -> Weight {
-		(1_296_000 as Weight)
+		(1_345_000 as Weight)
 	}
 	fn remark_with_event(b: u32, ) -> Weight {
-		(13_474_000 as Weight)
+		(9_697_000 as Weight)
 			// Standard Error: 0
 			.saturating_add((1_000 as Weight).saturating_mul(b as Weight))
 	}
 	fn set_heap_pages() -> Weight {
-		(2_024_000 as Weight)
+		(2_070_000 as Weight)
 			.saturating_add(RocksDbWeight::get().writes(1 as Weight))
 	}
 	fn set_changes_trie_config() -> Weight {
-		(10_551_000 as Weight)
+		(10_111_000 as Weight)
 			.saturating_add(RocksDbWeight::get().reads(1 as Weight))
 			.saturating_add(RocksDbWeight::get().writes(2 as Weight))
 	}
 	fn set_storage(i: u32, ) -> Weight {
 		(0 as Weight)
 			// Standard Error: 0
-			.saturating_add((612_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add((619_000 as Weight).saturating_mul(i as Weight))
 			.saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
 	}
 	fn kill_storage(i: u32, ) -> Weight {
-		(562_000 as Weight)
+		(1_647_000 as Weight)
 			// Standard Error: 0
-			.saturating_add((442_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add((460_000 as Weight).saturating_mul(i as Weight))
 			.saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
 	}
 	fn kill_prefix(p: u32, ) -> Weight {
-		(10_499_000 as Weight)
-			// Standard Error: 1_000
-			.saturating_add((840_000 as Weight).saturating_mul(p as Weight))
+		(10_678_000 as Weight)
+			// Standard Error: 0
+			.saturating_add((862_000 as Weight).saturating_mul(p as Weight))
 			.saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(p as Weight)))
 	}
 }