Sub-commands for benchmark (#11164)

* Restructure benchmark commands Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add benchmark block test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fixup imports Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * CI Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Extend error message Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Apply suggestions from code review Co-authored-by: Zeke Mostov <z.mostov@gmail.com> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add commands to node-template Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
2026-04-26 13:27:57 +00:00 · 2022-04-07 21:33:11 +02:00
parent ef5c4b7fc3
commit a7261180ee
33 changed files with 690 additions and 342 deletions
@@ -0,0 +1,65 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Code that is shared among all benchmarking sub-commands.
+
+pub mod record;
+pub mod stats;
+pub mod weight_params;
+
+pub use record::BenchRecord;
+pub use stats::{StatSelect, Stats};
+pub use weight_params::WeightParams;
+
+/// A Handlebars helper to add an underscore after every 3rd character,
+/// i.e. a separator for large numbers.
+#[derive(Clone, Copy)]
+pub struct UnderscoreHelper;
+
+impl handlebars::HelperDef for UnderscoreHelper {
+	fn call<'reg: 'rc, 'rc>(
+		&self,
+		h: &handlebars::Helper,
+		_: &handlebars::Handlebars,
+		_: &handlebars::Context,
+		_rc: &mut handlebars::RenderContext,
+		out: &mut dyn handlebars::Output,
+	) -> handlebars::HelperResult {
+		use handlebars::JsonRender;
+		let param = h.param(0).unwrap();
+		let underscore_param = underscore(param.value().render());
+		out.write(&underscore_param)?;
+		Ok(())
+	}
+}
+
+/// Add an underscore after every 3rd character, i.e. a separator for large numbers.
+fn underscore<Number>(i: Number) -> String
+where
+	Number: std::string::ToString,
+{
+	let mut s = String::new();
+	let i_str = i.to_string();
+	let a = i_str.chars().rev().enumerate();
+	for (idx, val) in a {
+		if idx != 0 && idx % 3 == 0 {
+			s.insert(0, '_');
+		}
+		s.insert(0, val);
+	}
+	s
+}
@@ -0,0 +1,72 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`].
+
+use sc_cli::Result;
+use sc_service::Configuration;
+
+use log::info;
+use serde::Serialize;
+use std::{fs, path::PathBuf, time::Duration};
+
+use super::Stats;
+
+/// Raw output of a Storage benchmark.
+#[derive(Debug, Default, Clone, Serialize)]
+pub struct BenchRecord {
+	/// Multi-Map of value sizes and the time that it took to access them.
+	ns_per_size: Vec<(u64, u64)>,
+}
+
+impl BenchRecord {
+	/// Appends a new record. Uses safe casts.
+	pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
+		let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
+		let ns: u64 = d
+			.as_nanos()
+			.try_into()
+			.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
+		self.ns_per_size.push((size, ns));
+		Ok(())
+	}
+
+	/// Returns the statistics for *time* and *value size*.
+	pub fn calculate_stats(self) -> Result<(Stats, Stats)> {
+		let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
+		let size = Stats::new(&size)?;
+		let time = Stats::new(&time)?;
+		Ok((time, size)) // The swap of time/size here is intentional.
+	}
+
+	/// Unless a path is specified, saves the raw results in a json file in the current directory.
+	/// Prefixes it with the DB name and suffixed with `path_suffix`.
+	pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
+		let mut path = PathBuf::from(out_path);
+		if path.is_dir() || path.as_os_str().is_empty() {
+			path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
+			path.set_extension("json");
+		}
+
+		let json = serde_json::to_string_pretty(&self)
+			.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
+
+		fs::write(&path, json)?;
+		info!("Raw data written to {:?}", fs::canonicalize(&path)?);
+		Ok(())
+	}
+}
@@ -0,0 +1,188 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Handles statistics that were generated from benchmarking results and
+//! that can be used to fill out weight templates.
+
+use sc_cli::Result;
+
+use serde::Serialize;
+use std::{fmt, result, str::FromStr};
+
+/// Various statistics that help to gauge the quality of the produced weights.
+/// Will be written to the weight file and printed to console.
+#[derive(Serialize, Default, Clone)]
+pub struct Stats {
+	/// Sum of all values.
+	pub sum: u64,
+	/// Minimal observed value.
+	pub min: u64,
+	/// Maximal observed value.
+	pub max: u64,
+
+	/// Average of all values.
+	pub avg: u64,
+	/// Median of all values.
+	pub median: u64,
+	/// Standard derivation of all values.
+	pub stddev: f64,
+
+	/// 99th percentile. At least 99% of all values are below this threshold.
+	pub p99: u64,
+	/// 95th percentile. At least 95% of all values are below this threshold.
+	pub p95: u64,
+	/// 75th percentile. At least 75% of all values are below this threshold.
+	pub p75: u64,
+}
+
+/// Selects a specific field from a [`Stats`] object.
+/// Not all fields are available.
+#[derive(Debug, Clone, Copy, Serialize, PartialEq)]
+pub enum StatSelect {
+	/// Select the maximum.
+	Maximum,
+	/// Select the average.
+	Average,
+	/// Select the median.
+	Median,
+	/// Select the 99th percentile.
+	P99Percentile,
+	/// Select the 95th percentile.
+	P95Percentile,
+	/// Select the 75th percentile.
+	P75Percentile,
+}
+
+impl Stats {
+	/// Calculates statistics and returns them.
+	pub fn new(xs: &Vec<u64>) -> Result<Self> {
+		if xs.is_empty() {
+			return Err("Empty input is invalid".into())
+		}
+		let (avg, stddev) = Self::avg_and_stddev(&xs);
+
+		Ok(Self {
+			sum: xs.iter().sum(),
+			min: *xs.iter().min().expect("Checked for non-empty above"),
+			max: *xs.iter().max().expect("Checked for non-empty above"),
+
+			avg: avg as u64,
+			median: Self::percentile(xs.clone(), 0.50),
+			stddev: (stddev * 100.0).round() / 100.0, // round to 1/100
+
+			p99: Self::percentile(xs.clone(), 0.99),
+			p95: Self::percentile(xs.clone(), 0.95),
+			p75: Self::percentile(xs.clone(), 0.75),
+		})
+	}
+
+	/// Returns the selected stat.
+	pub fn select(&self, s: StatSelect) -> u64 {
+		match s {
+			StatSelect::Maximum => self.max,
+			StatSelect::Average => self.avg,
+			StatSelect::Median => self.median,
+			StatSelect::P99Percentile => self.p99,
+			StatSelect::P95Percentile => self.p95,
+			StatSelect::P75Percentile => self.p75,
+		}
+	}
+
+	/// Returns the *average* and the *standard derivation*.
+	fn avg_and_stddev(xs: &Vec<u64>) -> (f64, f64) {
+		let avg = xs.iter().map(|x| *x as f64).sum::<f64>() / xs.len() as f64;
+		let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::<f64>() / xs.len() as f64;
+		(avg, variance.sqrt())
+	}
+
+	/// Returns the specified percentile for the given data.
+	/// This is best effort since it ignores the interpolation case.
+	fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
+		xs.sort();
+		let index = (xs.len() as f64 * p).ceil() as usize - 1;
+		xs[index.clamp(0, xs.len() - 1)]
+	}
+}
+
+impl fmt::Debug for Stats {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "Total: {}\n", self.sum)?;
+		write!(f, "Min: {}, Max: {}\n", self.min, self.max)?;
+		write!(f, "Average: {}, Median: {}, Stddev: {}\n", self.avg, self.median, self.stddev)?;
+		write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75)
+	}
+}
+
+impl Default for StatSelect {
+	/// Returns the `Average` selector.
+	fn default() -> Self {
+		Self::Average
+	}
+}
+
+impl FromStr for StatSelect {
+	type Err = &'static str;
+
+	fn from_str(day: &str) -> result::Result<Self, Self::Err> {
+		match day.to_lowercase().as_str() {
+			"max" => Ok(Self::Maximum),
+			"average" => Ok(Self::Average),
+			"median" => Ok(Self::Median),
+			"p99" => Ok(Self::P99Percentile),
+			"p95" => Ok(Self::P95Percentile),
+			"p75" => Ok(Self::P75Percentile),
+			_ => Err("String was not a StatSelect"),
+		}
+	}
+}
+
+#[cfg(test)]
+mod test_stats {
+	use super::Stats;
+	use rand::{seq::SliceRandom, thread_rng};
+
+	#[test]
+	fn stats_correct() {
+		let mut data: Vec<u64> = (1..=100).collect();
+		data.shuffle(&mut thread_rng());
+		let stats = Stats::new(&data).unwrap();
+
+		assert_eq!(stats.sum, 5050);
+		assert_eq!(stats.min, 1);
+		assert_eq!(stats.max, 100);
+
+		assert_eq!(stats.avg, 50);
+		assert_eq!(stats.median, 50); // 50.5 to be exact.
+		assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
+
+		assert_eq!(stats.p99, 99);
+		assert_eq!(stats.p95, 95);
+		assert_eq!(stats.p75, 75);
+	}
+
+	#[test]
+	fn no_panic_short_lengths() {
+		// Empty input does error.
+		assert!(Stats::new(&vec![]).is_err());
+
+		// Different small input lengths are fine.
+		for l in 1..10 {
+			let data = (0..=l).collect();
+			assert!(Stats::new(&data).is_ok());
+		}
+	}
+}
@@ -0,0 +1,95 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Calculates a weight from the [`super::Stats`] of a benchmark result.
+
+use sc_cli::Result;
+
+use clap::Args;
+use serde::Serialize;
+use std::path::PathBuf;
+
+use super::{StatSelect, Stats};
+
+/// Configures the weight generation.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct WeightParams {
+	/// File or directory to write the *weight* files to.
+	///
+	/// For Substrate this should be `frame/support/src/weights`.
+	#[clap(long)]
+	pub weight_path: Option<PathBuf>,
+
+	/// Select a specific metric to calculate the final weight output.
+	#[clap(long = "metric", default_value = "average")]
+	pub weight_metric: StatSelect,
+
+	/// Multiply the resulting weight with the given factor. Must be positive.
+	///
+	/// Is applied before `weight_add`.
+	#[clap(long = "mul", default_value = "1")]
+	pub weight_mul: f64,
+
+	/// Add the given offset to the resulting weight.
+	///
+	/// Is applied after `weight_mul`.
+	#[clap(long = "add", default_value = "0")]
+	pub weight_add: u64,
+}
+
+/// Calculates the final weight by multiplying the selected metric with
+/// `weight_mul` and adding `weight_add`.
+/// Does not use safe casts and can overflow.
+impl WeightParams {
+	pub fn calc_weight(&self, stat: &Stats) -> Result<u64> {
+		if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() {
+			return Err("invalid floating number for `weight_mul`".into())
+		}
+		let s = stat.select(self.weight_metric) as f64;
+		let w = s.mul_add(self.weight_mul, self.weight_add as f64).ceil();
+		Ok(w as u64) // No safe cast here since there is no `From<f64>` for `u64`.
+	}
+}
+
+#[cfg(test)]
+mod test_weight_params {
+	use super::WeightParams;
+	use crate::shared::{StatSelect, Stats};
+
+	#[test]
+	fn calc_weight_works() {
+		let stats = Stats { avg: 113, ..Default::default() };
+		let params = WeightParams {
+			weight_metric: StatSelect::Average,
+			weight_mul: 0.75,
+			weight_add: 3,
+			..Default::default()
+		};
+
+		let want = (113.0f64 * 0.75 + 3.0).ceil() as u64; // Ceil for overestimation.
+		let got = params.calc_weight(&stats).unwrap();
+		assert_eq!(want, got);
+	}
+
+	#[test]
+	fn calc_weight_detects_negative_mul() {
+		let stats = Stats::default();
+		let params = WeightParams { weight_mul: -0.75, ..Default::default() };
+
+		assert!(params.calc_weight(&stats).is_err());
+	}
+}