Create Benchmarking Setup for Identity Pallet #4695 (#4818)

* Starting

* closer

* Compiles!

* comments

* Create seperate mock

* Remove changes to test env

* Fix step calculation

* Add host function

* Add runtime api

* compiles

* Update to use offchain timestamp

* Gives a result

* added some CLI wip

* make generic

* Update instance

* Remove CLI stuff

* Remove last cli stuff

* undo more changes

* Update benchmarks

* Update Cargo.lock

* remove test

* Move loop out of runtime

* Benchmarking externalities

* Benchmarking state

* Implemented commit

* Make CLI work, move loop back into runtime

* Wipe resets to genesis

* Speedup benchmarks

* Use enum to select extrinsic within pallet

* CLI controls which module and extrinsic to call

* Select a pallet with cli

* Add steps and repeats to cli

* Output as CSV format

* Introduce benchmark pallet

* Append bench

* Use Results

* fix merge

* Clear Identity benchmark

* Bench request judgment and cancel request

* Add final benchmarks

* Fix CSV output

* Start cleaning up for PR

* Bump numbers in `wasmtime` integration tests.

* More docs

* Add rockdb feature to bench

* Fix formatting issues

* Add test feature to bench

* Add test feature to bench

* Add rocksdb feature flag

* Update bench.rs

Co-authored-by: Arkadiy Paronyan <arkady.paronyan@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>
This commit is contained in:
Shawn Tabrizi
2020-02-10 10:23:08 +01:00
committed by GitHub
parent 376deef36f
commit e5a7fcc8ea
21 changed files with 1217 additions and 19 deletions
+1
View File
@@ -58,6 +58,7 @@ use params::{
pub use params::{
SharedParams, ImportParams, ExecutionStrategy, Subcommand, RunCmd, BuildSpecCmd,
ExportBlocksCmd, ImportBlocksCmd, CheckBlockCmd, PurgeChainCmd, RevertCmd,
BenchmarkCmd,
};
pub use traits::GetSharedParams;
use app_dirs::{AppInfo, AppDataType};
+77
View File
@@ -845,6 +845,49 @@ pub struct PurgeChainCmd {
pub shared_params: SharedParams,
}
/// The `benchmark` command used to benchmark FRAME Pallets.
#[derive(Debug, StructOpt, Clone)]
pub struct BenchmarkCmd {
/// Select a FRAME Pallet to benchmark.
#[structopt(short, long)]
pub pallet: String,
/// Select an extrinsic to benchmark.
#[structopt(short, long)]
pub extrinsic: String,
/// Select how many samples we should take across the variable components.
#[structopt(short, long, default_value = "1")]
pub steps: u32,
/// Select how many repetitions of this benchmark should run.
#[structopt(short, long, default_value = "1")]
pub repeat: u32,
#[allow(missing_docs)]
#[structopt(flatten)]
pub shared_params: SharedParams,
/// The execution strategy that should be used for benchmarks
#[structopt(
long = "execution",
value_name = "STRATEGY",
possible_values = &ExecutionStrategy::variants(),
case_insensitive = true,
)]
pub execution: Option<ExecutionStrategy>,
/// Method for executing Wasm runtime code.
#[structopt(
long = "wasm-execution",
value_name = "METHOD",
possible_values = &WasmExecutionMethod::enabled_variants(),
case_insensitive = true,
default_value = "Interpreted"
)]
pub wasm_method: WasmExecutionMethod,
}
/// All core commands that are provided by default.
///
/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From
@@ -869,6 +912,9 @@ pub enum Subcommand {
/// Remove the whole chain data.
PurgeChain(PurgeChainCmd),
/// Run runtime benchmarks.
Benchmark(BenchmarkCmd),
}
impl Subcommand {
@@ -883,6 +929,7 @@ impl Subcommand {
CheckBlock(params) => &params.shared_params,
Revert(params) => &params.shared_params,
PurgeChain(params) => &params.shared_params,
Benchmark(params) => &params.shared_params,
}
}
@@ -909,6 +956,7 @@ impl Subcommand {
Subcommand::ImportBlocks(cmd) => cmd.run(config, builder),
Subcommand::CheckBlock(cmd) => cmd.run(config, builder),
Subcommand::PurgeChain(cmd) => cmd.run(config),
Subcommand::Benchmark(cmd) => cmd.run(config, builder),
Subcommand::Revert(cmd) => cmd.run(config, builder),
}
}
@@ -1186,3 +1234,32 @@ impl RevertCmd {
Ok(())
}
}
impl BenchmarkCmd {
/// Runs the command and benchmarks the chain.
pub fn run<G, E, B, BC, BB>(
self,
config: Configuration<G, E>,
_builder: B,
) -> error::Result<()>
where
B: FnOnce(Configuration<G, E>) -> Result<BC, sc_service::error::Error>,
G: RuntimeGenesis,
E: ChainSpecExtension,
BC: ServiceBuilderCommand<Block = BB> + Unpin,
BB: sp_runtime::traits::Block + Debug,
<<<BB as BlockT>::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug,
<BB as BlockT>::Hash: std::str::FromStr,
{
let spec = config.chain_spec.expect("chain_spec is always Some");
let execution_strategy = self.execution.unwrap_or(ExecutionStrategy::Native).into();
let wasm_method = self.wasm_method.into();
let pallet = self.pallet;
let extrinsic = self.extrinsic;
let steps = self.steps;
let repeat = self.repeat;
sc_service::chain_ops::benchmark_runtime::<BB, BC::NativeDispatch, _, _>(spec, execution_strategy, wasm_method, pallet, extrinsic, steps, repeat)?;
Ok(())
}
}
+1
View File
@@ -8,6 +8,7 @@ license = "GPL-3.0"
[dependencies]
parking_lot = "0.10.0"
log = "0.4.8"
rand = "0.7"
kvdb = "0.4.0"
kvdb-rocksdb = { version = "0.5", optional = true }
kvdb-memorydb = "0.4.0"
+286
View File
@@ -0,0 +1,286 @@
// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! State backend that's useful for benchmarking
use std::sync::Arc;
use std::path::PathBuf;
use std::cell::{Cell, RefCell};
use rand::Rng;
use hash_db::{Prefix, Hasher};
use sp_trie::{MemoryDB, prefixed_key};
use sp_core::storage::ChildInfo;
use sp_runtime::traits::{Block as BlockT, HasherFor};
use sp_runtime::Storage;
use sp_state_machine::{DBValue, backend::Backend as StateBackend};
use kvdb::{KeyValueDB, DBTransaction};
use kvdb_rocksdb::{Database, DatabaseConfig};
type DbState<B> = sp_state_machine::TrieBackend<
Arc<dyn sp_state_machine::Storage<HasherFor<B>>>, HasherFor<B>
>;
struct StorageDb<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
_block: std::marker::PhantomData<Block>,
}
impl<Block: BlockT> sp_state_machine::Storage<HasherFor<Block>> for StorageDb<Block> {
fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result<Option<DBValue>, String> {
let key = prefixed_key::<HasherFor<Block>>(key, prefix);
self.db.get(0, &key)
.map_err(|e| format!("Database backend error: {:?}", e))
}
}
/// State that manages the backend database reference. Allows runtime to control the database.
pub struct BenchmarkingState<B: BlockT> {
path: PathBuf,
root: Cell<B::Hash>,
state: RefCell<Option<DbState<B>>>,
db: Cell<Option<Arc<dyn KeyValueDB>>>,
genesis: <DbState<B> as StateBackend<HasherFor<B>>>::Transaction,
}
impl<B: BlockT> BenchmarkingState<B> {
/// Create a new instance that creates a database in a temporary dir.
pub fn new(genesis: Storage) -> Result<Self, String> {
let temp_dir = PathBuf::from(std::env::temp_dir());
let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect();
let path = temp_dir.join(&name);
let mut root = B::Hash::default();
let mut mdb = MemoryDB::<HasherFor<B>>::default();
sp_state_machine::TrieDBMut::<HasherFor<B>>::new(&mut mdb, &mut root);
std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?;
let mut state = BenchmarkingState {
state: RefCell::new(None),
db: Cell::new(None),
path,
root: Cell::new(root),
genesis: Default::default(),
};
state.reopen()?;
let child_delta = genesis.children.into_iter().map(|(storage_key, child_content)| (
storage_key,
child_content.data.into_iter().map(|(k, v)| (k, Some(v))),
child_content.child_info
));
let (root, transaction) = state.state.borrow_mut().as_mut().unwrap().full_storage_root(
genesis.top.into_iter().map(|(k, v)| (k, Some(v))),
child_delta,
);
state.genesis = transaction.clone();
state.commit(root, transaction)?;
Ok(state)
}
fn reopen(&self) -> Result<(), String> {
*self.state.borrow_mut() = None;
self.db.set(None);
let db_config = DatabaseConfig::with_columns(1);
let path = self.path.to_str()
.ok_or_else(|| String::from("Invalid database path"))?;
let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?);
self.db.set(Some(db.clone()));
let storage_db = Arc::new(StorageDb::<B> { db, _block: Default::default() });
*self.state.borrow_mut() = Some(DbState::<B>::new(storage_db, self.root.get()));
Ok(())
}
fn kill(&self) -> Result<(), String> {
self.db.set(None);
*self.state.borrow_mut() = None;
let mut root = B::Hash::default();
let mut mdb = MemoryDB::<HasherFor<B>>::default();
sp_state_machine::TrieDBMut::<HasherFor<B>>::new(&mut mdb, &mut root);
self.root.set(root);
std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into())
}
}
impl<B: BlockT> Drop for BenchmarkingState<B> {
fn drop(&mut self) {
self.kill().ok();
}
}
fn state_err() -> String {
"State is not open".into()
}
impl<B: BlockT> StateBackend<HasherFor<B>> for BenchmarkingState<B> {
type Error = <DbState<B> as StateBackend<HasherFor<B>>>::Error;
type Transaction = <DbState<B> as StateBackend<HasherFor<B>>>::Transaction;
type TrieBackendStorage = <DbState<B> as StateBackend<HasherFor<B>>>::TrieBackendStorage;
fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key)
}
fn storage_hash(&self, key: &[u8]) -> Result<Option<B::Hash>, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key)
}
fn child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(storage_key, child_info, key)
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key)
}
fn exists_child_storage(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<bool, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(storage_key, child_info, key)
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key)
}
fn next_child_storage_key(
&self,
storage_key: &[u8],
child_info: ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(storage_key, child_info, key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
if let Some(ref state) = *self.state.borrow() {
state.for_keys_with_prefix(prefix, f)
}
}
fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
if let Some(ref state) = *self.state.borrow() {
state.for_key_values_with_prefix(prefix, f)
}
}
fn for_keys_in_child_storage<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
f: F,
) {
if let Some(ref state) = *self.state.borrow() {
state.for_keys_in_child_storage(storage_key, child_info, f)
}
}
fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
f: F,
) {
if let Some(ref state) = *self.state.borrow() {
state.for_child_keys_with_prefix(storage_key, child_info, prefix, f)
}
}
fn storage_root<I>(&self, delta: I) -> (B::Hash, Self::Transaction) where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
{
self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta))
}
fn child_storage_root<I>(
&self,
storage_key: &[u8],
child_info: ChildInfo,
delta: I,
) -> (B::Hash, bool, Self::Transaction) where
I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
{
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(storage_key, child_info, delta))
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.pairs())
}
fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.keys(prefix))
}
fn child_keys(
&self,
storage_key: &[u8],
child_info: ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(storage_key, child_info, prefix))
}
fn as_trie_backend(&mut self)
-> Option<&sp_state_machine::TrieBackend<Self::TrieBackendStorage, HasherFor<B>>>
{
None
}
fn commit(&self, storage_root: <HasherFor<B> as Hasher>::Out, mut transaction: Self::Transaction)
-> Result<(), Self::Error>
{
if let Some(db) = self.db.take() {
let mut db_transaction = DBTransaction::new();
for (key, (val, rc)) in transaction.drain() {
if rc > 0 {
db_transaction.put(0, &key, &val);
} else if rc < 0 {
db_transaction.delete(0, &key);
}
}
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
self.root.set(storage_root);
} else {
return Err("Trying to commit to a closed db".into())
}
self.reopen()
}
fn wipe(&self) -> Result<(), Self::Error> {
self.kill()?;
self.reopen()?;
self.commit(self.root.get(), self.genesis.clone())?;
Ok(())
}
}
impl<Block: BlockT> std::fmt::Debug for BenchmarkingState<Block> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "DB at {:?}", self.path)
}
}
+6
View File
@@ -29,6 +29,9 @@
pub mod light;
pub mod offchain;
#[cfg(any(feature = "kvdb-rocksdb", test))]
pub mod bench;
mod children;
mod cache;
mod changes_tries_storage;
@@ -80,6 +83,9 @@ use crate::stats::StateUsageStats;
use log::{trace, debug, warn};
pub use sc_state_db::PruningMode;
#[cfg(any(feature = "kvdb-rocksdb", test))]
pub use bench::BenchmarkingState;
#[cfg(feature = "test-helpers")]
use sc_client::in_mem::Backend as InMemoryBackend;
@@ -88,7 +88,7 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) {
#[cfg(feature = "wasmtime")]
WasmExecutionMethod::Compiled => assert_eq!(
&format!("{:?}", e),
"Other(\"call to undefined external function with index 68\")"
"Other(\"call to undefined external function with index 71\")"
),
}
}
@@ -117,7 +117,7 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) {
#[cfg(feature = "wasmtime")]
WasmExecutionMethod::Compiled => assert_eq!(
&format!("{:?}", e),
"Other(\"call to undefined external function with index 69\")"
"Other(\"call to undefined external function with index 72\")"
),
}
}
+2
View File
@@ -673,6 +673,8 @@ impl<TBl, TRtApi, TGen, TCSExt, TCl, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TEx
pub trait ServiceBuilderCommand {
/// Block type this API operates on.
type Block: BlockT;
/// Native execution dispatch required by some commands.
type NativeDispatch: NativeExecutionDispatch + 'static;
/// Starts the process of importing blocks.
fn import_blocks(
self,
+67 -6
View File
@@ -22,14 +22,20 @@ use crate::error::Error;
use sc_chain_spec::{ChainSpec, RuntimeGenesis, Extension};
use log::{warn, info};
use futures::{future, prelude::*};
use sp_runtime::traits::{
Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion
use sp_runtime::{
BuildStorage, BenchmarkResults,
traits::{
Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion
}
};
use sp_runtime::generic::{BlockId, SignedBlock};
use codec::{Decode, Encode, IoReader};
use sc_client::Client;
use sc_client::{Client, ExecutionStrategy, StateMachine, LocalCallExecutor};
#[cfg(feature = "rocksdb")]
use sc_client_db::BenchmarkingState;
use sp_consensus::import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue};
use sp_consensus::BlockOrigin;
use sc_executor::{NativeExecutor, NativeExecutionDispatch, WasmExecutionMethod};
use std::{io::{Read, Write, Seek}, pin::Pin};
@@ -43,21 +49,76 @@ pub fn build_spec<G, E>(spec: ChainSpec<G, E>, raw: bool) -> error::Result<Strin
Ok(spec.to_json(raw)?)
}
/// Run runtime benchmarks.
#[cfg(feature = "rocksdb")]
pub fn benchmark_runtime<TBl, TExecDisp, G, E> (
spec: ChainSpec<G, E>,
strategy: ExecutionStrategy,
wasm_method: WasmExecutionMethod,
pallet: String,
extrinsic: String,
steps: u32,
repeat: u32,
) -> error::Result<()> where
TBl: BlockT,
TExecDisp: NativeExecutionDispatch + 'static,
G: RuntimeGenesis,
E: Extension,
{
let genesis_storage = spec.build_storage()?;
let mut changes = Default::default();
let state = BenchmarkingState::<TBl>::new(genesis_storage)?;
let executor = NativeExecutor::<TExecDisp>::new(
wasm_method,
None, // heap pages
);
let result = StateMachine::<_, _, NumberFor<TBl>, _>::new(
&state,
None,
&mut changes,
&executor,
"Benchmark_dispatch_benchmark",
&(&pallet, &extrinsic, steps, repeat).encode(),
Default::default(),
).execute(strategy).map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
let results = <Option<Vec<BenchmarkResults>> as Decode>::decode(&mut &result[..]).unwrap_or(None);
if let Some(results) = results {
// Print benchmark metadata
println!("Pallet: {:?}, Extrinsic: {:?}, Steps: {:?}, Repeat: {:?}", pallet, extrinsic, steps, repeat);
// Print the table header
results[0].0.iter().for_each(|param| print!("{:?},", param.0));
print!("time\n");
// Print the values
results.iter().for_each(|result| {
let parameters = &result.0;
parameters.iter().for_each(|param| print!("{:?},", param.1));
print!("{:?}\n", result.1);
});
info!("Done.");
} else {
info!("No Results.");
}
Ok(())
}
impl<
TBl, TRtApi, TGen, TCSExt, TBackend,
TExec, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP,
TExecDisp, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP,
TExPool, TRpc, Backend
> ServiceBuilderCommand for ServiceBuilder<
TBl, TRtApi, TGen, TCSExt, Client<TBackend, TExec, TBl, TRtApi>,
TBl, TRtApi, TGen, TCSExt,
Client<TBackend, LocalCallExecutor<TBackend, NativeExecutor<TExecDisp>>, TBl, TRtApi>,
TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TExPool, TRpc, Backend
> where
TBl: BlockT,
TBackend: 'static + sc_client_api::backend::Backend<TBl> + Send,
TExec: 'static + sc_client::CallExecutor<TBl> + Send + Sync + Clone,
TExecDisp: 'static + NativeExecutionDispatch,
TImpQu: 'static + ImportQueue<TBl>,
TRtApi: 'static + Send + Sync,
{
type Block = TBl;
type NativeDispatch = TExecDisp;
fn import_blocks(
self,
+1
View File
@@ -65,6 +65,7 @@ pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoo
pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions;
pub use sc_client::FinalityNotifications;
pub use sc_rpc::Metadata as RpcMetadata;
pub use sc_executor::NativeExecutionDispatch;
#[doc(hidden)]
pub use std::{ops::Deref, result::Result, sync::Arc};
#[doc(hidden)]
+1 -1
View File
@@ -104,4 +104,4 @@ pub use crate::{
},
leaves::LeafSet,
};
pub use sp_state_machine::{ExecutionStrategy, StorageProof};
pub use sp_state_machine::{ExecutionStrategy, StorageProof, StateMachine};