Sub-commands for benchmark (#11164)

* Restructure benchmark commands Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add benchmark block test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fixup imports Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * CI Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Extend error message Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Apply suggestions from code review Co-authored-by: Zeke Mostov <z.mostov@gmail.com> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add commands to node-template Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
2026-05-09 22:18:00 +00:00 · 2022-04-07 21:33:11 +02:00
parent ef5c4b7fc3
commit a7261180ee
33 changed files with 690 additions and 342 deletions
@@ -0,0 +1,188 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Handles statistics that were generated from benchmarking results and
+//! that can be used to fill out weight templates.
+
+use sc_cli::Result;
+
+use serde::Serialize;
+use std::{fmt, result, str::FromStr};
+
+/// Various statistics that help to gauge the quality of the produced weights.
+/// Will be written to the weight file and printed to console.
+#[derive(Serialize, Default, Clone)]
+pub struct Stats {
+	/// Sum of all values.
+	pub sum: u64,
+	/// Minimal observed value.
+	pub min: u64,
+	/// Maximal observed value.
+	pub max: u64,
+
+	/// Average of all values.
+	pub avg: u64,
+	/// Median of all values.
+	pub median: u64,
+	/// Standard derivation of all values.
+	pub stddev: f64,
+
+	/// 99th percentile. At least 99% of all values are below this threshold.
+	pub p99: u64,
+	/// 95th percentile. At least 95% of all values are below this threshold.
+	pub p95: u64,
+	/// 75th percentile. At least 75% of all values are below this threshold.
+	pub p75: u64,
+}
+
+/// Selects a specific field from a [`Stats`] object.
+/// Not all fields are available.
+#[derive(Debug, Clone, Copy, Serialize, PartialEq)]
+pub enum StatSelect {
+	/// Select the maximum.
+	Maximum,
+	/// Select the average.
+	Average,
+	/// Select the median.
+	Median,
+	/// Select the 99th percentile.
+	P99Percentile,
+	/// Select the 95th percentile.
+	P95Percentile,
+	/// Select the 75th percentile.
+	P75Percentile,
+}
+
+impl Stats {
+	/// Calculates statistics and returns them.
+	pub fn new(xs: &Vec<u64>) -> Result<Self> {
+		if xs.is_empty() {
+			return Err("Empty input is invalid".into())
+		}
+		let (avg, stddev) = Self::avg_and_stddev(&xs);
+
+		Ok(Self {
+			sum: xs.iter().sum(),
+			min: *xs.iter().min().expect("Checked for non-empty above"),
+			max: *xs.iter().max().expect("Checked for non-empty above"),
+
+			avg: avg as u64,
+			median: Self::percentile(xs.clone(), 0.50),
+			stddev: (stddev * 100.0).round() / 100.0, // round to 1/100
+
+			p99: Self::percentile(xs.clone(), 0.99),
+			p95: Self::percentile(xs.clone(), 0.95),
+			p75: Self::percentile(xs.clone(), 0.75),
+		})
+	}
+
+	/// Returns the selected stat.
+	pub fn select(&self, s: StatSelect) -> u64 {
+		match s {
+			StatSelect::Maximum => self.max,
+			StatSelect::Average => self.avg,
+			StatSelect::Median => self.median,
+			StatSelect::P99Percentile => self.p99,
+			StatSelect::P95Percentile => self.p95,
+			StatSelect::P75Percentile => self.p75,
+		}
+	}
+
+	/// Returns the *average* and the *standard derivation*.
+	fn avg_and_stddev(xs: &Vec<u64>) -> (f64, f64) {
+		let avg = xs.iter().map(|x| *x as f64).sum::<f64>() / xs.len() as f64;
+		let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::<f64>() / xs.len() as f64;
+		(avg, variance.sqrt())
+	}
+
+	/// Returns the specified percentile for the given data.
+	/// This is best effort since it ignores the interpolation case.
+	fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
+		xs.sort();
+		let index = (xs.len() as f64 * p).ceil() as usize - 1;
+		xs[index.clamp(0, xs.len() - 1)]
+	}
+}
+
+impl fmt::Debug for Stats {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "Total: {}\n", self.sum)?;
+		write!(f, "Min: {}, Max: {}\n", self.min, self.max)?;
+		write!(f, "Average: {}, Median: {}, Stddev: {}\n", self.avg, self.median, self.stddev)?;
+		write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75)
+	}
+}
+
+impl Default for StatSelect {
+	/// Returns the `Average` selector.
+	fn default() -> Self {
+		Self::Average
+	}
+}
+
+impl FromStr for StatSelect {
+	type Err = &'static str;
+
+	fn from_str(day: &str) -> result::Result<Self, Self::Err> {
+		match day.to_lowercase().as_str() {
+			"max" => Ok(Self::Maximum),
+			"average" => Ok(Self::Average),
+			"median" => Ok(Self::Median),
+			"p99" => Ok(Self::P99Percentile),
+			"p95" => Ok(Self::P95Percentile),
+			"p75" => Ok(Self::P75Percentile),
+			_ => Err("String was not a StatSelect"),
+		}
+	}
+}
+
+#[cfg(test)]
+mod test_stats {
+	use super::Stats;
+	use rand::{seq::SliceRandom, thread_rng};
+
+	#[test]
+	fn stats_correct() {
+		let mut data: Vec<u64> = (1..=100).collect();
+		data.shuffle(&mut thread_rng());
+		let stats = Stats::new(&data).unwrap();
+
+		assert_eq!(stats.sum, 5050);
+		assert_eq!(stats.min, 1);
+		assert_eq!(stats.max, 100);
+
+		assert_eq!(stats.avg, 50);
+		assert_eq!(stats.median, 50); // 50.5 to be exact.
+		assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
+
+		assert_eq!(stats.p99, 99);
+		assert_eq!(stats.p95, 95);
+		assert_eq!(stats.p75, 75);
+	}
+
+	#[test]
+	fn no_panic_short_lengths() {
+		// Empty input does error.
+		assert!(Stats::new(&vec![]).is_err());
+
+		// Different small input lengths are fine.
+		for l in 1..10 {
+			let data = (0..=l).collect();
+			assert!(Stats::new(&data).is_ok());
+		}
+	}
+}