From 7285cbc8016ba03a8d0abe0ff07cfdbff11bdddf Mon Sep 17 00:00:00 2001 From: Nikolay Volf Date: Fri, 13 Mar 2020 15:01:44 +0300 Subject: [PATCH] Update master benchmarks (#5230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * try more samples * try increase span * reaping bench + config * wip * ed25519 block import * update naming * align names * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * Update bin/node/testing/benches/import.rs Co-Authored-By: Bastian Köcher * use existential deposit Co-authored-by: Bastian Köcher --- substrate/bin/node/testing/benches/import.rs | 72 ++++++++++++-- substrate/bin/node/testing/src/bench.rs | 98 ++++++++++++++++---- 2 files changed, 143 insertions(+), 27 deletions(-) diff --git a/substrate/bin/node/testing/benches/import.rs b/substrate/bin/node/testing/benches/import.rs index 79cb71b164..b36d2e1181 100644 --- a/substrate/bin/node/testing/benches/import.rs +++ b/substrate/bin/node/testing/benches/import.rs @@ -29,7 +29,7 @@ //! It is not supposed to measure runtime modules weight correctness use std::fmt; -use node_testing::bench::{BenchDb, Profile}; +use node_testing::bench::{BenchDb, Profile, BlockType, KeyTypes}; use node_primitives::Block; use sp_runtime::generic::BlockId; use criterion::{Criterion, criterion_group, criterion_main}; @@ -37,8 +37,8 @@ use sc_client_api::backend::Backend; criterion_group!( name = benches; - config = Criterion::default().sample_size(20).warm_up_time(std::time::Duration::from_secs(20)); - targets = bench_block_import + config = Criterion::default().sample_size(50).warm_up_time(std::time::Duration::from_secs(20)); + targets = bench_block_import, bench_account_reaping, bench_account_ed25519 ); criterion_group!( name = wasm_size; @@ -57,8 +57,8 @@ fn bench_block_import(c: &mut Criterion) { // for future uses, uncomment if something wrong. // sc_cli::init_logger("sc_client=debug"); - let mut bench_db = BenchDb::new(128); - let block = bench_db.generate_block(100); + let mut bench_db = BenchDb::new(100); + let block = bench_db.generate_block(BlockType::RandomTransfers(100)); log::trace!( target: "bench-logistics", @@ -66,7 +66,7 @@ fn bench_block_import(c: &mut Criterion) { bench_db.path().display(), ); - c.bench_function_over_inputs("import block", + c.bench_function_over_inputs("import-block-B-0001", move |bencher, profile| { bencher.iter_batched( || { @@ -104,7 +104,61 @@ fn bench_block_import(c: &mut Criterion) { .expect("RocksDB backend always provides usage info!"), ); }, - criterion::BatchSize::PerIteration, + criterion::BatchSize::LargeInput, + ); + }, + vec![Profile::Wasm, Profile::Native], + ); +} + +fn bench_account_reaping(c: &mut Criterion) { + sc_cli::init_logger(""); + + let mut bench_db = BenchDb::new(100); + let block = bench_db.generate_block(BlockType::RandomTransfersReaping(100)); + + c.bench_function_over_inputs("import-block-reaping-B-0002", + move |bencher, profile| { + bencher.iter_batched( + || { + let context = bench_db.create_context(*profile); + + // mostly to just launch compiler before benching! + context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version"); + + context + }, + |mut context| { + context.import_block(block.clone()); + }, + criterion::BatchSize::LargeInput, + ); + }, + vec![Profile::Wasm, Profile::Native], + ); +} + +fn bench_account_ed25519(c: &mut Criterion) { + sc_cli::init_logger(""); + + let mut bench_db = BenchDb::with_key_types(100, KeyTypes::Ed25519); + let block = bench_db.generate_block(BlockType::RandomTransfers(100)); + + c.bench_function_over_inputs("import-block-ed25519-B-0003", + move |bencher, profile| { + bencher.iter_batched( + || { + let context = bench_db.create_context(*profile); + context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version"); + + context + }, + |mut context| { + context.import_block(block.clone()); + }, + criterion::BatchSize::LargeInput, ); }, vec![Profile::Wasm, Profile::Native], @@ -118,7 +172,7 @@ fn profile_block_import(c: &mut Criterion) { sc_cli::init_logger(""); let mut bench_db = BenchDb::new(128); - let block = bench_db.generate_block(100); + let block = bench_db.generate_block(BlockType::RandomTransfers(100)); c.bench_function("profile block", move |bencher| { @@ -174,7 +228,7 @@ impl Iterator for SetupIterator { let size = self.current * self.multiplier; let mut db = BenchDb::new(size); - let block = db.generate_block(size); + let block = db.generate_block(BlockType::RandomTransfers(size)); Some(Setup { db, block }) } } diff --git a/substrate/bin/node/testing/src/bench.rs b/substrate/bin/node/testing/src/bench.rs index 49949021b2..d49a6411cf 100644 --- a/substrate/bin/node/testing/src/bench.rs +++ b/substrate/bin/node/testing/src/bench.rs @@ -47,7 +47,7 @@ use node_runtime::{ AccountId, Signature, }; -use sp_core::ExecutionContext; +use sp_core::{ExecutionContext, blake2_256}; use sp_api::ProvideRuntimeApi; use sp_block_builder::BlockBuilder; use sp_inherents::InherentData; @@ -55,7 +55,7 @@ use sc_client_api::{ ExecutionStrategy, execution_extensions::{ExecutionExtensions, ExecutionStrategies}, }; -use sp_core::{Pair, Public, sr25519}; +use sp_core::{Pair, Public, sr25519, ed25519}; use sc_block_builder::BlockBuilderProvider; /// Keyring full of accounts for benching. @@ -67,7 +67,22 @@ use sc_block_builder::BlockBuilderProvider; /// //endowed-user//N #[derive(Clone)] pub struct BenchKeyring { - accounts: BTreeMap, + accounts: BTreeMap, +} + +#[derive(Clone)] +enum BenchPair { + Sr25519(sr25519::Pair), + Ed25519(ed25519::Pair), +} + +impl BenchPair { + fn sign(&self, payload: &[u8]) -> Signature { + match self { + Self::Sr25519(pair) => pair.sign(payload).into(), + Self::Ed25519(pair) => pair.sign(payload).into(), + } + } } /// Pre-initialized benchmarking database. @@ -109,17 +124,31 @@ impl Clone for BenchDb { } } +/// Type of block for generation +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum BlockType { + /// Bunch of random transfers. + RandomTransfers(usize), + /// Bunch of random transfers that drain all of the source balance. + RandomTransfersReaping(usize), +} + +impl BlockType { + /// Number of transactions for this block type. + pub fn transactions(&self) -> usize { + match self { + Self::RandomTransfers(v) | Self::RandomTransfersReaping(v) => *v, + } + } +} + impl BenchDb { /// New immutable benchmarking database. /// - /// This will generate database files in random temporary directory - /// and keep it there until struct is dropped. - /// - /// You can `clone` this database or you can `create_context` from it - /// (which also do `clone`) to run actual operation against new database - /// which will be identical to this. - pub fn new(keyring_length: usize) -> Self { - let keyring = BenchKeyring::new(keyring_length); + /// See [`new`] method documentation for more information about the purpose + /// of this structure. + pub fn with_key_types(keyring_length: usize, key_types: KeyTypes) -> Self { + let keyring = BenchKeyring::new(keyring_length, key_types); let dir = tempfile::tempdir().expect("temp dir creation failed"); log::trace!( @@ -133,6 +162,18 @@ impl BenchDb { BenchDb { keyring, directory_guard } } + /// New immutable benchmarking database. + /// + /// This will generate database files in random temporary directory + /// and keep it there until struct is dropped. + /// + /// You can `clone` this database or you can `create_context` from it + /// (which also do `clone`) to run actual operation against new database + /// which will be identical to this. + pub fn new(keyring_length: usize) -> Self { + Self::with_key_types(keyring_length, KeyTypes::Sr25519) + } + // This should return client that is doing everything that full node // is doing. // @@ -163,7 +204,7 @@ impl BenchDb { } /// Generate new block using this database. - pub fn generate_block(&mut self, transactions: usize) -> Block { + pub fn generate_block(&mut self, block_type: BlockType) -> Block { let (client, _backend) = Self::bench_client( self.directory_guard.path(), Profile::Wasm, @@ -203,7 +244,7 @@ impl BenchDb { let mut iteration = 0; let start = std::time::Instant::now(); - for _ in 0..transactions { + for _ in 0..block_type.transactions() { let sender = self.keyring.at(iteration); let receiver = get_account_id_from_seed::( @@ -212,11 +253,14 @@ impl BenchDb { let signed = self.keyring.sign( CheckedExtrinsic { - signed: Some((sender, signed_extra(0, 1*DOLLARS))), + signed: Some((sender, signed_extra(0, node_runtime::ExistentialDeposit::get() + 1))), function: Call::Balances( BalancesCall::transfer( pallet_indices::address::Address::Id(receiver), - 1*DOLLARS + match block_type { + BlockType::RandomTransfers(_) => node_runtime::ExistentialDeposit::get() + 1, + BlockType::RandomTransfersReaping(_) => 100*DOLLARS - node_runtime::ExistentialDeposit::get() - 1, + } ) ), }, @@ -267,17 +311,35 @@ impl BenchDb { } } +/// Key types to be used in benching keyring +pub enum KeyTypes { + /// sr25519 signing keys + Sr25519, + /// ed25519 signing keys + Ed25519, +} + impl BenchKeyring { /// New keyring. /// /// `length` is the number of accounts generated. - pub fn new(length: usize) -> Self { + pub fn new(length: usize, key_types: KeyTypes) -> Self { let mut accounts = BTreeMap::new(); for n in 0..length { let seed = format!("//endowed-user/{}", n); - let pair = sr25519::Pair::from_string(&seed, None).expect("failed to generate pair"); - let account_id = AccountPublic::from(pair.public()).into_account(); + let (account_id, pair) = match key_types { + KeyTypes::Sr25519 => { + let pair = sr25519::Pair::from_string(&seed, None).expect("failed to generate pair"); + let account_id = AccountPublic::from(pair.public()).into_account(); + (account_id, BenchPair::Sr25519(pair)) + }, + KeyTypes::Ed25519 => { + let pair = ed25519::Pair::from_seed(&blake2_256(seed.as_bytes())); + let account_id = AccountPublic::from(pair.public()).into_account(); + (account_id, BenchPair::Ed25519(pair)) + }, + }; accounts.insert(account_id, pair); }