diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index a54fe1f64c..b7e9322937 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -3314,6 +3314,21 @@ dependencies = [ "void", ] +[[package]] +name = "node-bench" +version = "0.8.0-alpha.5" +dependencies = [ + "log", + "node-primitives", + "node-testing", + "sc-cli", + "sc-client-api", + "serde", + "serde_json", + "sp-runtime", + "structopt", +] + [[package]] name = "node-cli" version = "2.0.0-alpha.5" diff --git a/substrate/Cargo.toml b/substrate/Cargo.toml index 9b30bfd39e..4fb7b58a43 100644 --- a/substrate/Cargo.toml +++ b/substrate/Cargo.toml @@ -3,6 +3,7 @@ members = [ "bin/node-template/node", "bin/node-template/runtime", "bin/node-template/pallets/template", + "bin/node/bench", "bin/node/cli", "bin/node/executor", "bin/node/primitives", diff --git a/substrate/bin/node/bench/Cargo.toml b/substrate/bin/node/bench/Cargo.toml new file mode 100644 index 0000000000..22e7fe51d8 --- /dev/null +++ b/substrate/bin/node/bench/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "node-bench" +version = "0.8.0-alpha.5" +authors = ["Parity Technologies "] +description = "Substrate node integration benchmarks." +edition = "2018" +license = "GPL-3.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +log = "0.4.8" +node-primitives = { version = "2.0.0-alpha.5", path = "../primitives" } +node-testing = { version = "2.0.0-alpha.5", path = "../testing" } +sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } +sc-client-api = { version = "2.0.0-alpha.5", path = "../../../client/api/" } +sp-runtime = { version = "2.0.0-alpha.5", path = "../../../primitives/runtime" } +serde = "1.0.101" +serde_json = "1.0.41" +structopt = "0.3" \ No newline at end of file diff --git a/substrate/bin/node/bench/src/core.rs b/substrate/bin/node/bench/src/core.rs new file mode 100644 index 0000000000..a8164db75a --- /dev/null +++ b/substrate/bin/node/bench/src/core.rs @@ -0,0 +1,140 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use std::{fmt, borrow::{Cow, ToOwned}}; +use serde::Serialize; + +pub struct Path(Vec); + +impl Path { + pub fn new(initial: &'static [&'static str]) -> Self { + Path(initial.iter().map(|x| x.to_string()).collect()) + } +} + +impl Path { + pub fn push(&mut self, item: &str) { + self.0.push(item.to_string()); + } + + pub fn full(&self) -> String { + self.0.iter().fold(String::new(), |mut val, next| { val.push_str("::"); val.push_str(next); val }) + } + + pub fn has(&self, path: &str) -> bool { + self.full().contains(path) + } +} + +pub trait BenchmarkDescription { + fn path(&self) -> Path; + + fn setup(self: Box) -> Box; + + fn name(&self) -> Cow<'static, str>; +} + +pub trait Benchmark { + fn run(&mut self) -> std::time::Duration; +} + +#[derive(Debug, Clone, Serialize)] +pub struct BenchmarkOutput { + name: String, + raw_average: u64, + average: u64, +} + +struct NsFormatter(u64); + +impl fmt::Display for NsFormatter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let v = self.0; + + if v < 100 { + return write!(f, "{} ns", v) + } + + if self.0 < 10_000 { + return write!(f, "{:.1} µs", v as f64 / 1000.0) + } + + if self.0 < 1_000_000 { + return write!(f, "{:.1} ms", v as f64 / 1_000_000.0) + } + + if self.0 < 100_000_000 { + return write!(f, "{} ms", v as f64 / 1_000_000.0) + } + + write!(f, "{:.2} s", v as f64 / 1_000_000_000.0) + } +} + +impl fmt::Display for BenchmarkOutput { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "({}: avg {}, w_avg {})", + self.name, + NsFormatter(self.raw_average), + NsFormatter(self.average), + ) + } +} + +pub fn run_benchmark(benchmark: Box) -> BenchmarkOutput { + let name = benchmark.name().to_owned(); + let mut benchmark = benchmark.setup(); + + let mut durations: Vec = vec![]; + for _ in 0..50 { + let duration = benchmark.run(); + durations.push(duration.as_nanos()); + } + + durations.sort(); + + let raw_average = (durations.iter().sum::() / (durations.len() as u128)) as u64; + let average = (durations.iter().skip(10).take(30).sum::() / 30) as u64; + + BenchmarkOutput { + name: name.into(), + raw_average, + average, + } +} + +macro_rules! matrix( + ( $var:ident in $over:expr => $tt:expr, $( $rest:tt )* ) => { + { + let mut res = Vec::>::new(); + for $var in $over.iter() { + res.push(Box::new($tt)); + } + res.extend(matrix!( $($rest)* )); + res + } + }; + ( $var:expr, $( $rest:tt )*) => { + { + let mut res = vec![Box::new($var) as Box]; + res.extend(matrix!( $($rest)* )); + res + } + }; + () => { vec![] } +); \ No newline at end of file diff --git a/substrate/bin/node/bench/src/import.rs b/substrate/bin/node/bench/src/import.rs new file mode 100644 index 0000000000..20181bf4c7 --- /dev/null +++ b/substrate/bin/node/bench/src/import.rs @@ -0,0 +1,136 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block import benchmark. +//! +//! This benchmark is expected to measure block import operation of +//! some more or less full block. +//! +//! As we also want to protect against cold-cache attacks, this +//! benchmark should not rely on any caching (except those that +//! DO NOT depend on user input). Thus block generation should be +//! based on randomized operation. +//! +//! This is supposed to be very simple benchmark and is not subject +//! to much configuring - just block full of randomized transactions. +//! It is not supposed to measure runtime modules weight correctness + +use std::borrow::Cow; + +use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes}; +use node_primitives::Block; +use sc_client_api::backend::Backend; +use sp_runtime::generic::BlockId; + +use crate::core::{self, Path}; + +#[derive(Clone, Copy, Debug)] +pub enum SizeType { Small, Medium, Large } + +impl SizeType { + fn transactions(&self) -> usize { + match self { + SizeType::Small => 10, + SizeType::Medium => 100, + SizeType::Large => 500, + } + } +} + +pub struct ImportBenchmarkDescription { + pub profile: Profile, + pub key_types: KeyTypes, + pub size: SizeType, +} + +pub struct ImportBenchmark { + profile: Profile, + database: BenchDb, + block: Block, +} + +impl core::BenchmarkDescription for ImportBenchmarkDescription { + fn path(&self) -> Path { + + let mut path = Path::new(&["node", "import"]); + + match self.profile { + Profile::Wasm => path.push("wasm"), + Profile::Native => path.push("native"), + } + + match self.key_types { + KeyTypes::Sr25519 => path.push("sr25519"), + KeyTypes::Ed25519 => path.push("ed25519"), + } + + match self.size { + SizeType::Small => path.push("small"), + SizeType::Medium => path.push("medium"), + SizeType::Large => path.push("large"), + } + + path + } + + fn setup(self: Box) -> Box { + let profile = self.profile; + let mut bench_db = BenchDb::with_key_types(self.size.transactions(), self.key_types); + let block = bench_db.generate_block(BlockType::RandomTransfers(self.size.transactions())); + Box::new(ImportBenchmark { + database: bench_db, + block, + profile, + }) + } + + fn name(&self) -> Cow<'static, str> { + match self.profile { + Profile::Wasm => "Import benchmark (random transfers, wasm)".into(), + Profile::Native => "Import benchmark (random transfers, native)".into(), + } + } +} + +impl core::Benchmark for ImportBenchmark { + fn run(&mut self) -> std::time::Duration { + let mut context = self.database.create_context(self.profile); + + let _ = context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version") + .spec_version; + + let start = std::time::Instant::now(); + context.import_block(self.block.clone()); + let elapsed = start.elapsed(); + + log::info!( + target: "bench-logistics", + "imported block with {} tx, took: {:#?}", + self.block.extrinsics.len(), + elapsed, + ); + + log::info!( + target: "bench-logistics", + "usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ); + + elapsed + } +} \ No newline at end of file diff --git a/substrate/bin/node/bench/src/main.rs b/substrate/bin/node/bench/src/main.rs new file mode 100644 index 0000000000..8f04546526 --- /dev/null +++ b/substrate/bin/node/bench/src/main.rs @@ -0,0 +1,95 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +#[macro_use] mod core; +mod import; + +use crate::core::run_benchmark; +use import::{ImportBenchmarkDescription, SizeType}; +use node_testing::bench::{Profile, KeyTypes}; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(name = "node-bench", about = "Node integration benchmarks")] +struct Opt { + /// Show list of all available benchmarks. + /// + /// Will output ("name", "path"). Benchmarks can then be filtered by path. + #[structopt(short, long)] + list: bool, + + /// Machine readable json output. + /// + /// This also suppresses all regular output (except to stderr) + #[structopt(short, long)] + json: bool, + + /// Filter benchmarks. + /// + /// Run with `--list` for the hint of what to filter. + filter: Option, +} + +fn main() { + let opt = Opt::from_args(); + + if !opt.json { + sc_cli::init_logger(""); + } + + let benchmarks = matrix!( + profile in [Profile::Wasm, Profile::Native] => + ImportBenchmarkDescription { + profile: *profile, + key_types: KeyTypes::Sr25519, + size: SizeType::Medium, + }, + ImportBenchmarkDescription { + profile: Profile::Native, + key_types: KeyTypes::Ed25519, + size: SizeType::Medium, + }, + size in [SizeType::Small, SizeType::Large] => + ImportBenchmarkDescription { + profile: Profile::Native, + key_types: KeyTypes::Sr25519, + size: *size, + }, + ); + + if opt.list { + for benchmark in benchmarks.iter() { + log::info!("{}: {}", benchmark.name(), benchmark.path().full()) + } + return; + } + + let mut results = Vec::new(); + for benchmark in benchmarks { + if opt.filter.as_ref().map(|f| benchmark.path().has(f)).unwrap_or(true) { + log::info!("Starting {}", benchmark.name()); + let result = run_benchmark(benchmark); + log::info!("{}", result); + + results.push(result); + } + } + + if opt.json { + let json_result: String = serde_json::to_string(&results).expect("Failed to construct json"); + println!("{}", json_result); + } +} \ No newline at end of file diff --git a/substrate/bin/node/testing/Cargo.toml b/substrate/bin/node/testing/Cargo.toml index b16e313574..df73e20070 100644 --- a/substrate/bin/node/testing/Cargo.toml +++ b/substrate/bin/node/testing/Cargo.toml @@ -53,9 +53,5 @@ criterion = "0.3.0" sc-cli = { version = "0.8.0-alpha.5", path = "../../../client/cli" } sc-service = { version = "0.8.0-alpha.5", path = "../../../client/service", features = ["rocksdb"] } -[[bench]] -name = "import" -harness = false - [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/bin/node/testing/benches/import.rs b/substrate/bin/node/testing/benches/import.rs deleted file mode 100644 index b36d2e1181..0000000000 --- a/substrate/bin/node/testing/benches/import.rs +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Block import benchmark. -//! -//! This benchmark is expected to measure block import operation of -//! some more or less full block. -//! -//! As we also want to protect against cold-cache attacks, this -//! benchmark should not rely on any caching (except those that -//! DO NOT depend on user input). Thus block generation should be -//! based on randomized operation. -//! -//! This is supposed to be very simple benchmark and is not subject -//! to much configuring - just block full of randomized transactions. -//! It is not supposed to measure runtime modules weight correctness - -use std::fmt; -use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes}; -use node_primitives::Block; -use sp_runtime::generic::BlockId; -use criterion::{Criterion, criterion_group, criterion_main}; -use sc_client_api::backend::Backend; - -criterion_group!( - name = benches; - config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20)); - targets = bench_block_import, bench_account_reaping, bench_account_ed25519 -); -criterion_group!( - name = wasm_size; - config = Criterion::default().sample_size(10); - targets = bench_wasm_size_import -); -criterion_group!( - name = profile; - config = Criterion::default().sample_size(10); - targets = profile_block_import -); -criterion_main!(benches, profile); - -fn bench_block_import(c: &mut Criterion) { - sc_cli::init_logger(""); - // for future uses, uncomment if something wrong. - // sc_cli::init_logger("sc_client=debug"); - - let mut bench_db = BenchDb::new(100); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - log::trace!( - target: "bench-logistics", - "Seed database directory: {}", - bench_db.path().display(), - ); - - c.bench_function_over_inputs("import-block-B-0001", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - - // mostly to just launch compiler before benching! - let version = context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version") - .spec_version; - - log::trace!( - target: "bench-logistics", - "Next iteration database directory: {}, runtime version: {}", - context.path().display(), version, - ); - - context - }, - |mut context| { - let start = std::time::Instant::now(); - context.import_block(block.clone()); - let elapsed = start.elapsed(); - - log::info!( - target: "bench-logistics", - "imported block with {} tx, took: {:#?}", - block.extrinsics.len(), - elapsed, - ); - - log::info!( - target: "bench-logistics", - "usage info: {}", - context.backend.usage_info() - .expect("RocksDB backend always provides usage info!"), - ); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -fn bench_account_reaping(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::new(100); - let block = bench_db.generate_block(BlockType::RandomTransfersReaping(100)); - - c.bench_function_over_inputs("import-block-reaping-B-0002", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - - // mostly to just launch compiler before benching! - context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version"); - - context - }, - |mut context| { - context.import_block(block.clone()); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -fn bench_account_ed25519(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::with_key_types(100, KeyTypes::Ed25519); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - c.bench_function_over_inputs("import-block-ed25519-B-0003", - move |bencher, profile| { - bencher.iter_batched( - || { - let context = bench_db.create_context(*profile); - context.client.runtime_version_at(&BlockId::Number(0)) - .expect("Failed to get runtime version"); - - context - }, - |mut context| { - context.import_block(block.clone()); - }, - criterion::BatchSize::LargeInput, - ); - }, - vec![Profile::Wasm, Profile::Native], - ); -} - -// This is not an actual benchmark, so don't use it to measure anything. -// It just produces special pattern of cpu load that allows easy picking -// the part of block import for the profiling in the tool of choice. -fn profile_block_import(c: &mut Criterion) { - sc_cli::init_logger(""); - - let mut bench_db = BenchDb::new(128); - let block = bench_db.generate_block(BlockType::RandomTransfers(100)); - - c.bench_function("profile block", - move |bencher| { - bencher.iter_batched( - || { - bench_db.create_context(Profile::Native) - }, - |mut context| { - // until better osx signpost/callgrind signal is possible to use - // in rust, we just pause everything completely to help choosing - // actual profiling interval - std::thread::park_timeout(std::time::Duration::from_secs(2)); - context.import_block(block.clone()); - // and here as well - std::thread::park_timeout(std::time::Duration::from_secs(2)); - log::info!( - target: "bench-logistics", - "imported block, usage info: {}", - context.backend.usage_info() - .expect("RocksDB backend always provides usage info!"), - ) - }, - criterion::BatchSize::PerIteration, - ); - }, - ); -} - -struct Setup { - db: BenchDb, - block: Block, -} - -struct SetupIterator { - current: usize, - finish: usize, - multiplier: usize, -} - -impl SetupIterator { - fn new(current: usize, finish: usize, multiplier: usize) -> Self { - SetupIterator { current, finish, multiplier } - } -} - -impl Iterator for SetupIterator { - type Item = Setup; - - fn next(&mut self) -> Option { - if self.current >= self.finish { return None } - - self.current += 1; - - let size = self.current * self.multiplier; - let mut db = BenchDb::new(size); - let block = db.generate_block(BlockType::RandomTransfers(size)); - Some(Setup { db, block }) - } -} - -impl fmt::Debug for Setup { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Setup: {} tx/block", self.block.extrinsics.len()) - } -} - -fn bench_wasm_size_import(c: &mut Criterion) { - sc_cli::init_logger(""); - - c.bench_function_over_inputs("wasm_size_import", - move |bencher, setup| { - bencher.iter_batched( - || { - setup.db.create_context(Profile::Wasm) - }, - |mut context| { - context.import_block(setup.block.clone()); - }, - criterion::BatchSize::PerIteration, - ); - }, - SetupIterator::new(5, 15, 50), - ); -}