Add execution overhead benchmarking (#10977)

* Add benchmark-block Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Remove first approach This reverts commit cf96a0a2307433f23187e77864de4a89ecbaef0a. * Add block and extrinsic benchmarks * Doc Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fix template Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Beauty fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Check for non-empty chain Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add tests for Stats Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Apply suggestions from code review Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Push first version again Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Push first version again Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Cleanup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Beauty fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Apply suggestions from code review Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> * Update utils/frame/benchmarking-cli/src/overhead/template.rs Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Doc + Template fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Comment fix Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Pust merge fixup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fixup Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Move code to better place Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com> Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
2026-04-26 12:17:58 +00:00 · 2022-03-17 11:40:31 +01:00
parent 26a8c7e6b2
commit 96cf135586
17 changed files with 852 additions and 53 deletions
@@ -33,8 +33,8 @@ use serde::Serialize;
 use sp_runtime::generic::BlockId;
 use std::{fmt::Debug, path::PathBuf, sync::Arc};

-use super::{record::StatSelect, template::TemplateData};
-
+use super::template::TemplateData;
+use crate::post_processing::WeightParams;
 /// Benchmark the storage of a Substrate node with a live chain snapshot.
 #[derive(Debug, Parser)]
 pub struct StorageCmd {
@@ -58,24 +58,9 @@ pub struct StorageCmd {
 /// Parameters for modifying the benchmark behaviour and the post processing of the results.
 #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
 pub struct StorageParams {
-	/// Path to write the *weight* file to. Can be a file or directory.
-	/// For substrate this should be `frame/support/src/weights`.
-	#[clap(long)]
-	pub weight_path: Option<PathBuf>,
-
-	/// Select a specific metric to calculate the final weight output.
-	#[clap(long = "metric", default_value = "average")]
-	pub weight_metric: StatSelect,
-
-	/// Multiply the resulting weight with the given factor. Must be positive.
-	/// Is calculated before `weight_add`.
-	#[clap(long = "mul", default_value = "1")]
-	pub weight_mul: f64,
-
-	/// Add the given offset to the resulting weight.
-	/// Is calculated after `weight_mul`.
-	#[clap(long = "add", default_value = "0")]
-	pub weight_add: u64,
+	#[allow(missing_docs)]
+	#[clap(flatten)]
+	pub weight_params: WeightParams,

 	/// Skip the `read` benchmark.
 	#[clap(long)]
@@ -153,7 +138,7 @@ impl StorageCmd {
 			template.set_stats(None, Some(stats))?;
 		}

-		template.write(&self.params.weight_path, &self.params.template_path)
+		template.write(&self.params.weight_params.weight_path, &self.params.template_path)
 	}

 	/// Returns the specified state version.
@@ -36,25 +36,25 @@ pub(crate) struct BenchRecord {
 #[derive(Serialize, Default, Clone)]
 pub(crate) struct Stats {
 	/// Sum of all values.
-	sum: u64,
+	pub(crate) sum: u64,
 	/// Minimal observed value.
-	min: u64,
+	pub(crate) min: u64,
 	/// Maximal observed value.
-	max: u64,
+	pub(crate) max: u64,

 	/// Average of all values.
-	avg: u64,
+	pub(crate) avg: u64,
 	/// Median of all values.
-	median: u64,
+	pub(crate) median: u64,
 	/// Standard derivation of all values.
-	stddev: f64,
+	pub(crate) stddev: f64,

 	/// 99th percentile. At least 99% of all values are below this threshold.
-	p99: u64,
+	pub(crate) p99: u64,
 	/// 95th percentile. At least 95% of all values are below this threshold.
-	p95: u64,
+	pub(crate) p95: u64,
 	/// 75th percentile. At least 75% of all values are below this threshold.
-	p75: u64,
+	pub(crate) p75: u64,
 }

 /// Selects a specific field from a [`Stats`] object.
@@ -159,8 +159,8 @@ impl Stats {
 	/// This is best effort since it ignores the interpolation case.
 	fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
 		xs.sort();
-		let index = (xs.len() as f64 * p).ceil() as usize;
-		xs[index]
+		let index = (xs.len() as f64 * p).ceil() as usize - 1;
+		xs[index.clamp(0, xs.len() - 1)]
 	}
 }

@@ -195,3 +195,40 @@ impl FromStr for StatSelect {
 		}
 	}
 }
+
+#[cfg(test)]
+mod test_stats {
+	use super::Stats;
+	use rand::{seq::SliceRandom, thread_rng};
+
+	#[test]
+	fn stats_correct() {
+		let mut data: Vec<u64> = (1..=100).collect();
+		data.shuffle(&mut thread_rng());
+		let stats = Stats::new(&data).unwrap();
+
+		assert_eq!(stats.sum, 5050);
+		assert_eq!(stats.min, 1);
+		assert_eq!(stats.max, 100);
+
+		assert_eq!(stats.avg, 50);
+		assert_eq!(stats.median, 50); // 50.5 to be exact.
+		assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
+
+		assert_eq!(stats.p99, 99);
+		assert_eq!(stats.p95, 95);
+		assert_eq!(stats.p75, 75);
+	}
+
+	#[test]
+	fn no_panic_short_lengths() {
+		// Empty input does error.
+		assert!(Stats::new(&vec![]).is_err());
+
+		// Different small input lengths are fine.
+		for l in 1..10 {
+			let data = (0..=l).collect();
+			assert!(Stats::new(&data).is_ok());
+		}
+	}
+}
@@ -77,11 +77,11 @@ impl TemplateData {
 		write: Option<(Stats, Stats)>,
 	) -> Result<()> {
 		if let Some(read) = read {
-			self.read_weight = calc_weight(&read.0, &self.params)?;
+			self.read_weight = self.params.weight_params.calc_weight(&read.0)?;
 			self.read = Some(read);
 		}
 		if let Some(write) = write {
-			self.write_weight = calc_weight(&write.0, &self.params)?;
+			self.write_weight = self.params.weight_params.calc_weight(&write.0)?;
 			self.write = Some(write);
 		}
 		Ok(())
@@ -130,15 +130,3 @@ impl TemplateData {
 		path
 	}
 }
-
-/// Calculates the final weight by multiplying the selected metric with
-/// `mul` and adding `add`.
-/// Does not use safe casts and can overflow.
-fn calc_weight(stat: &Stats, params: &StorageParams) -> Result<u64> {
-	if params.weight_mul.is_sign_negative() || !params.weight_mul.is_normal() {
-		return Err("invalid floating number for `weight_mul`".into())
-	}
-	let s = stat.select(params.weight_metric) as f64;
-	let w = s.mul_add(params.weight_mul, params.weight_add as f64).ceil();
-	Ok(w as u64) // No safe cast here since there is no `From<f64>` for `u64`.
-}