Use custom runner for import benchmarks (#5477)

* custom benchmark runner

* add license preamble

* delete old benchmarks

* update toml

* imporove macro

* tabs

* add size

* size fixes

* add docs for cli

substrate/bin/node/bench/src/core.rs

@@ -0,0 +1,140 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{fmt, borrow::{Cow, ToOwned}};
+use serde::Serialize;
+
+pub struct Path(Vec<String>);
+
+impl Path {
+	pub fn new(initial: &'static [&'static str]) -> Self {
+		Path(initial.iter().map(|x| x.to_string()).collect())
+	}
+}
+
+impl Path {
+	pub fn push(&mut self, item: &str) {
+		self.0.push(item.to_string());
+	}
+
+	pub fn full(&self) -> String {
+		self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val })
+	}
+
+	pub fn has(&self, path: &str) -> bool {
+		self.full().contains(path)
+	}
+}
+
+pub trait BenchmarkDescription {
+	fn path(&self) -> Path;
+
+	fn setup(self: Box<Self>) -> Box<dyn Benchmark>;
+
+	fn name(&self) -> Cow<'static, str>;
+}
+
+pub trait Benchmark {
+	fn run(&mut self) -> std::time::Duration;
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct BenchmarkOutput {
+	name: String,
+	raw_average: u64,
+	average: u64,
+}
+
+struct NsFormatter(u64);
+
+impl fmt::Display for NsFormatter {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		let v = self.0;
+
+		if v < 100 {
+			return write!(f, "{} ns", v)
+		}
+
+		if self.0 < 10_000 {
+			return write!(f, "{:.1} µs", v as f64 / 1000.0)
+		}
+
+		if self.0 < 1_000_000 {
+			return write!(f, "{:.1} ms", v as f64 / 1_000_000.0)
+		}
+
+		if self.0 < 100_000_000 {
+			return write!(f, "{} ms", v as f64 / 1_000_000.0)
+		}
+
+		write!(f, "{:.2} s", v as f64 / 1_000_000_000.0)
+	}
+}
+
+impl fmt::Display for BenchmarkOutput {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+			f,
+			"({}: avg {}, w_avg {})",
+			self.name,
+			NsFormatter(self.raw_average),
+			NsFormatter(self.average),
+		)
+    }
+}
+
+pub fn run_benchmark(benchmark: Box<dyn BenchmarkDescription>) -> BenchmarkOutput {
+	let name = benchmark.name().to_owned();
+	let mut benchmark = benchmark.setup();
+
+	let mut durations: Vec<u128> = vec![];
+	for _ in 0..50 {
+		let duration = benchmark.run();
+		durations.push(duration.as_nanos());
+	}
+
+	durations.sort();
+
+	let raw_average = (durations.iter().sum::<u128>() / (durations.len() as u128)) as u64;
+	let average = (durations.iter().skip(10).take(30).sum::<u128>() / 30) as u64;
+
+	BenchmarkOutput {
+		name: name.into(),
+		raw_average,
+		average,
+	}
+}
+
+macro_rules! matrix(
+	( $var:ident in $over:expr => $tt:expr,  $( $rest:tt )* ) => {
+		{
+			let mut res = Vec::<Box<dyn crate::core::BenchmarkDescription>>::new();
+			for $var in $over.iter() {
+				res.push(Box::new($tt));
+			}
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	( $var:expr, $( $rest:tt )*) => {
+		{
+			let mut res = vec![Box::new($var) as Box<dyn crate::core::BenchmarkDescription>];
+			res.extend(matrix!( $($rest)* ));
+			res
+		}
+	};
+	() => { vec![] }
+);

substrate/bin/node/bench/src/import.rs

@@ -0,0 +1,136 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block import benchmark.
+//!
+//! This benchmark is expected to measure block import operation of
+//! some more or less full block.
+//!
+//! As we also want to protect against cold-cache attacks, this
+//! benchmark should not rely on any caching (except those that
+//! DO NOT depend on user input). Thus block generation should be
+//! based on randomized operation.
+//!
+//! This is supposed to be very simple benchmark and is not subject
+//! to much configuring - just block full of randomized transactions.
+//! It is not supposed to measure runtime modules weight correctness
+
+use std::borrow::Cow;
+
+use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes};
+use node_primitives::Block;
+use sc_client_api::backend::Backend;
+use sp_runtime::generic::BlockId;
+
+use crate::core::{self, Path};
+
+#[derive(Clone, Copy, Debug)]
+pub enum SizeType { Small, Medium, Large }
+
+impl SizeType {
+	fn transactions(&self) -> usize {
+		match self {
+			SizeType::Small => 10,
+			SizeType::Medium => 100,
+			SizeType::Large => 500,
+		}
+	}
+}
+
+pub struct ImportBenchmarkDescription {
+	pub profile: Profile,
+	pub key_types: KeyTypes,
+	pub size: SizeType,
+}
+
+pub struct ImportBenchmark {
+	profile: Profile,
+	database: BenchDb,
+	block: Block,
+}
+
+impl core::BenchmarkDescription for ImportBenchmarkDescription {
+	fn path(&self) -> Path {
+
+		let mut path = Path::new(&["node", "import"]);
+
+		match self.profile {
+			Profile::Wasm => path.push("wasm"),
+			Profile::Native => path.push("native"),
+		}
+
+		match self.key_types {
+			KeyTypes::Sr25519 => path.push("sr25519"),
+			KeyTypes::Ed25519 => path.push("ed25519"),
+		}
+
+		match self.size {
+			SizeType::Small => path.push("small"),
+			SizeType::Medium => path.push("medium"),
+			SizeType::Large => path.push("large"),
+		}
+
+		path
+	}
+
+	fn setup(self: Box<Self>) -> Box<dyn core::Benchmark> {
+		let profile = self.profile;
+		let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types);
+		let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions()));
+		Box::new(ImportBenchmark {
+			database: bench_db,
+			block,
+			profile,
+		})
+	}
+
+	fn name(&self) -> Cow<'static, str> {
+		match self.profile {
+			Profile::Wasm => "Import benchmark (random transfers, wasm)".into(),
+			Profile::Native => "Import benchmark (random transfers, native)".into(),
+		}
+	}
+}
+
+impl core::Benchmark for ImportBenchmark {
+	fn run(&mut self) -> std::time::Duration {
+		let mut context = self.database.create_context(self.profile);
+
+		let _ = context.client.runtime_version_at(&BlockId::Number(0))
+			.expect("Failed to get runtime version")
+			.spec_version;
+
+		let start = std::time::Instant::now();
+		context.import_block(self.block.clone());
+		let elapsed = start.elapsed();
+
+		log::info!(
+			target: "bench-logistics",
+			"imported block with {} tx, took: {:#?}",
+			self.block.extrinsics.len(),
+			elapsed,
+		);
+
+		log::info!(
+			target: "bench-logistics",
+			"usage info: {}",
+			context.backend.usage_info()
+				.expect("RocksDB backend always provides usage info!"),
+		);
+
+		elapsed
+	}
+}

substrate/bin/node/bench/src/main.rs

@@ -0,0 +1,95 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#[macro_use] mod core;
+mod import;
+
+use crate::core::run_benchmark;
+use import::{ImportBenchmarkDescription, SizeType};
+use node_testing::bench::{Profile, KeyTypes};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "node-bench", about = "Node integration benchmarks")]
+struct Opt {
+	/// Show list of all available benchmarks.
+	///
+	/// Will output ("name", "path"). Benchmarks can then be filtered by path.
+	#[structopt(short, long)]
+	list: bool,
+
+	/// Machine readable json output.
+	///
+	/// This also suppresses all regular output (except to stderr)
+	#[structopt(short, long)]
+	json: bool,
+
+	/// Filter benchmarks.
+	///
+	/// Run with `--list` for the hint of what to filter.
+	filter: Option<String>,
+}
+
+fn main() {
+	let opt = Opt::from_args();
+
+	if !opt.json {
+		sc_cli::init_logger("");
+	}
+
+	let benchmarks = matrix!(
+		profile in [Profile::Wasm, Profile::Native] =>
+			ImportBenchmarkDescription {
+				profile: *profile,
+				key_types: KeyTypes::Sr25519,
+				size: SizeType::Medium,
+			},
+		ImportBenchmarkDescription {
+			profile: Profile::Native,
+			key_types: KeyTypes::Ed25519,
+			size: SizeType::Medium,
+		},
+		size in [SizeType::Small, SizeType::Large] =>
+			ImportBenchmarkDescription {
+				profile: Profile::Native,
+				key_types: KeyTypes::Sr25519,
+				size: *size,
+			},
+	);
+
+	if opt.list {
+		for benchmark in benchmarks.iter() {
+			log::info!("{}: {}", benchmark.name(), benchmark.path().full())
+		}
+		return;
+	}
+
+	let mut results = Vec::new();
+	for benchmark in benchmarks {
+		if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) {
+			log::info!("Starting {}", benchmark.name());
+			let result = run_benchmark(benchmark);
+			log::info!("{}", result);
+
+			results.push(result);
+		}
+	}
+
+	if opt.json {
+		let json_result: String = serde_json::to_string(&results).expect("Failed to construct json");
+		println!("{}", json_result);
+	}
+}