A Pallet for Benchmarking Common Runtime Operations (#4902)

* Benchmark pallet

* Add a bunch more benchmarks

* do nothing test

* new benchmarks

* Clean up extra tests

* Encode and Decode Vec<T::AccountId>

* Starting to migrate benchmarks to macro

* Use macro

* Remove call and storage

* Update Cargo.toml

* Add storage recalc benchmark

* Add support for custom functions in benchmark! macro

* Reset DB for storage recalc

* Feedback from review

* Add more comments

* Remove benchmark pallet from node

* Fix cargo files

* Fix comments

* Change `crate` to `super`

* missed one

* Use results of benchmark encode/decode

* Pass generic to extra functions

* reset macro to master

* Update lib.rs

* Update to use standard syntax

substrate/Cargo.lock

@@ -3966,6 +3966,20 @@ dependencies = [
  "sp-std",
 ]
 
+[[package]]
+name = "pallet-benchmark"
+version = "2.0.0-alpha.3"
+dependencies = [
+ "frame-benchmarking",
+ "frame-support",
+ "frame-system",
+ "parity-scale-codec",
+ "serde",
+ "sp-io",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "pallet-collective"
 version = "2.0.0-alpha.3"

substrate/Cargo.toml

@@ -62,6 +62,7 @@ members = [
 	"frame/babe",
 	"frame/balances",
 	"frame/benchmarking",
+	"frame/benchmark",
 	"frame/collective",
 	"frame/contracts",
 	"frame/contracts/rpc",

substrate/client/network/src/protocol/sync/blocks.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::mem;
 use std::cmp;
 use std::ops::Range;
 use std::collections::{HashMap, BTreeMap};

substrate/frame/balances/src/benchmarking.rs

@@ -20,7 +20,7 @@ use super::*;
 
 use frame_system::RawOrigin;
 use frame_benchmarking::{benchmarks, account};
-use sp_runtime::traits::{Bounded, Dispatchable};
+use sp_runtime::traits::Bounded;
 
 use crate::Module as Balances;
 

substrate/frame/benchmark/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "pallet-benchmark"
+version = "2.0.0-alpha.3"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0"
+
+[dependencies]
+serde = { version = "1.0.101", optional = true }
+codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] }
+sp-std = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/std" }
+sp-io = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/io" }
+sp-runtime = { version = "2.0.0-alpha.3", default-features = false, path = "../../primitives/runtime" }
+frame-support = { version = "2.0.0-alpha.3", default-features = false, path = "../support" }
+frame-system = { version = "2.0.0-alpha.3", default-features = false, path = "../system" }
+frame-benchmarking = { version = "2.0.0-alpha.3", default-features = false, path = "../benchmarking" }
+
+[features]
+default = ["std"]
+std = [
+	"serde",
+	"codec/std",
+	"sp-std/std",
+	"sp-io/std",
+	"sp-runtime/std",
+	"frame-support/std",
+	"frame-system/std",
+	"frame-benchmarking/std",
+]

substrate/frame/benchmark/src/benchmarking.rs

@@ -0,0 +1,131 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Benchmarks for common FRAME Pallet operations.
+
+use super::*;
+
+use frame_system::RawOrigin;
+use sp_std::prelude::*;
+use frame_benchmarking::{benchmarks, account};
+
+use crate::Module as Benchmark;
+
+const SEED: u32 = 0;
+
+benchmarks! {
+	_ {
+		let m in 1 .. 1000 => {
+			let origin = RawOrigin::Signed(account("member", m, SEED));
+			Benchmark::<T>::add_member_list(origin.into())?
+		};
+		let i in 1 .. 1000 => {
+			MyMap::insert(i, i);
+		};
+		let d in 1 .. 1000 => {
+			for i in 0..d {
+				for j in 0..100 {
+					MyDoubleMap::insert(i, j, d);
+				}
+			}
+		};
+	}
+
+	add_member_list {
+		let m in ...;
+	}: _(RawOrigin::Signed(account("member", m + 1, SEED)))
+
+	append_member_list {
+		let m in ...;
+	}: _(RawOrigin::Signed(account("member", m + 1, SEED)))
+
+	read_value {
+		let n in 1 .. 1000;
+		MyValue::put(n);
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), n)
+
+	put_value {
+		let n in 1 .. 1000;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), n)
+
+	exists_value {
+		let n in 1 .. 1000;
+		MyValue::put(n);
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), n)
+
+	remove_value {
+		let i in ...;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), i)
+
+	read_map {
+		let i in ...;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), i)
+
+	insert_map {
+		let n in 1 .. 1000;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), n)
+
+	contains_key_map {
+		let i in ...;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), i)
+
+	remove_prefix {
+		let d in ...;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), d)
+
+	do_nothing {
+		let n in 1 .. 1000;
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), n)
+
+	encode_accounts {
+		let a in 1 .. 1000;
+		let mut accounts = Vec::new();
+		for _ in 0..a {
+			accounts.push(account::<T::AccountId>("encode", a, SEED));
+		}
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), accounts)
+
+	decode_accounts {
+		let a in 1 .. 1000;
+		let mut accounts = Vec::new();
+		for _ in 0..a {
+			accounts.push(account::<T::AccountId>("encode", a, SEED));
+		}
+		let bytes = accounts.encode();
+	}: _(RawOrigin::Signed(account("user", 0, SEED)), bytes)
+
+	// Custom implementation to handle benchmarking of storage recalculation.
+	// Puts `repeat` number of items into random storage keys, and then times how
+	// long it takes to recalculate the storage root.
+	storage_root {
+		let z in 0 .. 10000;
+	}: {
+		for index in 0 .. z {
+			let random = (index).using_encoded(sp_io::hashing::blake2_256);
+			sp_io::storage::set(&random, &random);
+		}
+	}
+
+	// Custom implementation to handle benchmarking of calling a host function.
+	// Will check how long it takes to call `current_time()`.
+	current_time {
+		let z in 0 .. 1000;
+	}: {
+		for _ in 0 .. z {
+			let _ = frame_benchmarking::benchmarking::current_time();
+		}
+	}
+}

substrate/frame/benchmark/src/lib.rs

@@ -0,0 +1,177 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A pallet that contains common runtime patterns in an isolated manner.
+//! This pallet is **not** meant to be used in a production blockchain, just
+//! for benchmarking and testing purposes.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use frame_support::{decl_module, decl_storage, decl_event, decl_error};
+use frame_support::traits::Currency;
+use frame_system::{self as system, ensure_signed};
+use codec::{Encode, Decode};
+use sp_std::prelude::Vec;
+
+pub mod benchmarking;
+
+/// Type alias for currency balance.
+pub type BalanceOf<T> = <<T as Trait>::Currency as Currency<<T as frame_system::Trait>::AccountId>>::Balance;
+
+/// The pallet's configuration trait.
+pub trait Trait: system::Trait {
+	type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
+	type Currency: Currency<Self::AccountId>;
+}
+
+// This pallet's storage items.
+decl_storage! {
+	trait Store for Module<T: Trait> as Benchmark {
+		MyMemberList: Vec<T::AccountId>;
+		MyMemberMap: map hasher(blake2_256) T::AccountId => bool;
+		MyValue: u32;
+		MyMap: map hasher(blake2_256) u32 => u32;
+		MyDoubleMap: double_map hasher(blake2_256) u32, hasher(blake2_256) u32 => u32;
+	}
+}
+
+// The pallet's events
+decl_event!(
+	pub enum Event<T> where AccountId = <T as system::Trait>::AccountId {
+		Dummy(u32, AccountId),
+	}
+);
+
+// The pallet's errors
+decl_error! {
+	pub enum Error for Module<T: Trait> {
+	}
+}
+
+// The pallet's dispatchable functions.
+decl_module! {
+	/// The module declaration.
+	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+		type Error = Error<T>;
+
+		fn deposit_event() = default;
+
+		/// Do nothing.
+		pub fn do_nothing(_origin, input: u32) {
+			if input > 0 {
+				return Ok(());
+			}
+		}
+
+		/// Read a value from storage value `repeat` number of times.
+		/// Note the first `get()` read here will pull from the underlying
+		/// storage database, however, the `repeat` calls will all pull from the
+		/// storage overlay cache. You must consider this when analyzing the
+		/// results of the benchmark.
+		pub fn read_value(_origin, repeat: u32) {
+			for _ in 0..repeat {
+				MyValue::get();
+			}
+		}
+
+		/// Put a value into a storage value.
+		pub fn put_value(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyValue::put(r);
+			}
+		}
+
+		/// Read a value from storage `repeat` number of times.
+		/// Note the first `exists()` read here will pull from the underlying
+		/// storage database, however, the `repeat` calls will all pull from the
+		/// storage overlay cache. You must consider this when analyzing the
+		/// results of the benchmark.
+		pub fn exists_value(_origin, repeat: u32) {
+			for _ in 0..repeat {
+				MyValue::exists();
+			}
+		}
+
+		/// Remove a value from storage `repeat` number of times.
+		pub fn remove_value(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyMap::remove(r);
+			}
+		}
+
+		/// Read a value from storage map `repeat` number of times.
+		pub fn read_map(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyMap::get(r);
+			}
+		}
+
+		/// Insert a value into a map.
+		pub fn insert_map(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyMap::insert(r, r);
+			}
+		}
+
+		/// Check is a map contains a value `repeat` number of times.
+		pub fn contains_key_map(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyMap::contains_key(r);
+			}
+		}
+
+		/// Read a value from storage `repeat` number of times.
+		pub fn remove_prefix(_origin, repeat: u32) {
+			for r in 0..repeat {
+				MyDoubleMap::remove_prefix(r);
+			}
+		}
+
+		// Add user to the list.
+		pub fn add_member_list(origin) {
+			let who = ensure_signed(origin)?;
+			MyMemberList::<T>::mutate(|x| x.push(who));
+		}
+
+		// Append user to the list.
+		pub fn append_member_list(origin) {
+			let who = ensure_signed(origin)?;
+			MyMemberList::<T>::append(&[who])?;
+		}
+
+		// Encode a vector of accounts to bytes.
+		pub fn encode_accounts(_origin, accounts: Vec<T::AccountId>) {
+			let bytes = accounts.encode();
+
+			// In an attempt to tell the compiler not to optimize away this benchmark, we will use
+			// the result of encoding the accounts.
+			if bytes.is_empty() {
+				frame_support::print("You are encoding zero accounts.");
+			}
+		}
+
+		// Decode bytes into a vector of accounts.
+		pub fn decode_accounts(_origin, bytes: Vec<u8>) {
+			let accounts: Vec<T::AccountId> = Decode::decode(&mut bytes.as_slice()).map_err(|_| "Could not decode")?;
+
+			// In an attempt to tell the compiler not to optimize away this benchmark, we will use
+			// the result of decoding the bytes.
+			if accounts.is_empty() {
+				frame_support::print("You are decoding zero bytes.");
+			}
+		}
+	}
+}

substrate/frame/benchmarking/src/lib.rs

@@ -23,6 +23,7 @@ mod utils;
 pub use utils::*;
 #[doc(hidden)]
 pub use sp_io::storage::root as storage_root;
+pub use sp_runtime::traits::Dispatchable;
 
 /// Construct pallet benchmarks for weighing dispatchables.
 ///
@@ -79,14 +80,14 @@ pub use sp_io::storage::root as storage_root;
 ///   }
 ///
 ///   // first dispatchable: foo; this is a user dispatchable and operates on a `u8` vector of
-///   // size `l`, which we allow to be initialised as usual.
+///   // size `l`, which we allow to be initialized as usual.
 ///   foo {
 ///     let caller = account::<T>(b"caller", 0, benchmarks_seed);
 ///     let l = ...;
 ///   }: _(Origin::Signed(caller), vec![0u8; l])
 ///
 ///   // second dispatchable: bar; this is a root dispatchable and accepts a `u8` vector of size
-///   // `l`. We don't want it preininitialised like before so we override using the `=> ()` notation.
+///   // `l`. We don't want it pre-initialized like before so we override using the `=> ()` notation.
 ///   // In this case, we explicitly name the call using `bar` instead of `_`.
 ///   bar {
 ///     let l = _ .. _ => ();
@@ -157,7 +158,7 @@ macro_rules! benchmarks_iter {
 	) => {
 		$crate::benchmarks_iter! {
 			{ $( $common )* } ( $( $names )* ) $name { $( $code )* }: { 
-				Call::<T>::$dispatch($($arg),*).dispatch($origin.into())?;
+				<Call<T> as $crate::Dispatchable>::dispatch(Call::<T>::$dispatch($($arg),*), $origin.into())?;
 			} $( $rest )*
 		}
 	};

substrate/frame/example/src/lib.rs

@@ -263,7 +263,7 @@ use frame_benchmarking::{benchmarks, account};
 use frame_system::{self as system, ensure_signed, ensure_root, RawOrigin};
 use codec::{Encode, Decode};
 use sp_runtime::{
-	traits::{SignedExtension, Bounded, SaturatedConversion, Dispatchable},
+	traits::{SignedExtension, Bounded, SaturatedConversion},
 	transaction_validity::{
 		ValidTransaction, TransactionValidityError, InvalidTransaction, TransactionValidity,
 	},

substrate/frame/identity/src/benchmarking.rs

@@ -21,7 +21,7 @@ use super::*;
 use frame_system::RawOrigin;
 use sp_io::hashing::blake2_256;
 use frame_benchmarking::benchmarks;
-use sp_runtime::traits::{Bounded, Dispatchable};
+use sp_runtime::traits::Bounded;
 
 use crate::Module as Identity;
 

substrate/frame/timestamp/src/benchmarking.rs

@@ -22,7 +22,6 @@ use sp_std::prelude::*;
 
 use frame_system::RawOrigin;
 use frame_benchmarking::benchmarks;
-use sp_runtime::traits::Dispatchable;
 
 const MAX_TIME: u32 = 100;