Create Benchmarking Setup for Identity Pallet #4695 (#4818)

* Starting

* closer

* Compiles!

* comments

* Create seperate mock

* Remove changes to test env

* Fix step calculation

* Add host function

* Add runtime api

* compiles

* Update to use offchain timestamp

* Gives a result

* added some CLI wip

* make generic

* Update instance

* Remove CLI stuff

* Remove last cli stuff

* undo more changes

* Update benchmarks

* Update Cargo.lock

* remove test

* Move loop out of runtime

* Benchmarking externalities

* Benchmarking state

* Implemented commit

* Make CLI work, move loop back into runtime

* Wipe resets to genesis

* Speedup benchmarks

* Use enum to select extrinsic within pallet

* CLI controls which module and extrinsic to call

* Select a pallet with cli

* Add steps and repeats to cli

* Output as CSV format

* Introduce benchmark pallet

* Append bench

* Use Results

* fix merge

* Clear Identity benchmark

* Bench request judgment and cancel request

* Add final benchmarks

* Fix CSV output

* Start cleaning up for PR

* Bump numbers in `wasmtime` integration tests.

* More docs

* Add rockdb feature to bench

* Fix formatting issues

* Add test feature to bench

* Add test feature to bench

* Add rocksdb feature flag

* Update bench.rs

Co-authored-by: Arkadiy Paronyan <arkady.paronyan@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>

substrate/primitives/externalities/src/lib.rs

@@ -202,6 +202,14 @@ pub trait Externalities: ExtensionStore {
 	///
 	/// Returns the SCALE encoded hash.
 	fn storage_changes_root(&mut self, parent: &[u8]) -> Result<Option<Vec<u8>>, ()>;
+
+	fn wipe(&mut self) {
+		unimplemented!()
+	}
+
+	fn commit(&mut self) {
+		unimplemented!()
+	}
 }
 
 /// Extension for the [`Externalities`] trait.

substrate/primitives/io/src/lib.rs

@@ -771,6 +771,30 @@ pub trait Logging {
 	}
 }
 
+/// Interface that provides functions for benchmarking the runtime.
+#[runtime_interface]
+pub trait Benchmarking {
+		/// Get the number of nanoseconds passed since the UNIX epoch
+		///
+		/// WARNING! This is a non-deterministic call. Do not use this within
+		/// consensus critical logic.
+		fn current_time() -> u128 {
+			std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH)
+				.expect("Unix time doesn't go backwards; qed")
+				.as_nanos()
+		}
+
+		/// Reset the trie database to the genesis state.
+		fn wipe_db(&mut self) {
+			self.wipe()
+		}
+
+		/// Commit pending storage changes to the trie database and clear the database cache.
+		fn commit_db(&mut self) {
+			self.commit()
+		}
+}
+
 /// Wasm-only interface that provides functions for interacting with the sandbox.
 #[runtime_interface(wasm_only)]
 pub trait Sandbox {
@@ -923,6 +947,7 @@ pub type SubstrateHostFunctions = (
 	logging::HostFunctions,
 	sandbox::HostFunctions,
 	crate::trie::HostFunctions,
+	benchmarking::HostFunctions,
 );
 
 #[cfg(test)]

substrate/primitives/runtime/src/lib.rs

@@ -685,6 +685,18 @@ pub fn print(print: impl traits::Printable) {
 	print.print();
 }
 
+/// An alphabet of possible parameters to use for benchmarking.
+#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)]
+#[allow(missing_docs)]
+pub enum BenchmarkParameter {
+	A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z,
+}
+
+/// Results from running benchmarks on a FRAME pallet.
+/// Contains duration of the function call in nanoseconds along with the benchmark parameters
+/// used for that benchmark result.
+pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128);
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/primitives/runtime/src/traits.rs

@@ -24,6 +24,7 @@ use std::fmt::Display;
 #[cfg(feature = "std")]
 use serde::{Serialize, Deserialize, de::DeserializeOwned};
 use sp_core::{self, Hasher, Blake2Hasher, TypeId, RuntimeDebug};
+use crate::BenchmarkParameter;
 use crate::codec::{Codec, Encode, Decode};
 use crate::transaction_validity::{
 	ValidTransaction, TransactionValidity, TransactionValidityError, UnknownTransaction,
@@ -1328,6 +1329,26 @@ pub trait BlockIdTo<Block: self::Block> {
 	) -> Result<Option<NumberFor<Block>>, Self::Error>;
 }
 
+/// The pallet benchmarking trait.
+pub trait Benchmarking<T> {
+	/// Run the benchmarks for this pallet.
+	///
+	/// Parameters
+	/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
+	/// - `steps`: The number of sample points you want to take across the range of parameters.
+	/// - `repeat`: The number of times you want to repeat a benchmark.
+	fn run_benchmark(extrinsic: Vec<u8>, steps: u32, repeat: u32) -> Result<Vec<T>, &'static str>;
+}
+
+/// The required setup for creating a benchmark.
+pub trait BenchmarkingSetup<T, Call, RawOrigin> {
+	/// Return the components and their ranges which should be tested in this benchmark.
+	fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>;
+
+	/// Set up the storage, and prepare a call and caller to test in a single run of the benchmark.
+	fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result<(Call, RawOrigin), &'static str>;
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/primitives/state-machine/src/backend.rs

@@ -213,6 +213,16 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
 	fn usage_info(&self) -> UsageInfo {
 		UsageInfo::empty()
 	}
+
+	/// Wipe the state database.
+	fn wipe(&self) -> Result<(), Self::Error> {
+		unimplemented!()
+	}
+
+	/// Commit given transaction to storage.
+	fn commit(&self, _storage_root: H::Out, _transaction: Self::Transaction) -> Result<(), Self::Error> {
+		unimplemented!()
+	}
 }
 
 impl<'a, T: Backend<H>, H: Hasher> Backend<H> for &'a T {

substrate/primitives/state-machine/src/ext.rs

@@ -583,6 +583,25 @@ where
 
 		root.map(|r| r.map(|o| o.encode()))
 	}
+
+	fn wipe(&mut self) {
+		self.overlay.discard_prospective();
+		self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.storage_transaction_cache.reset();
+		self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL)
+	}
+
+	fn commit(&mut self) {
+		self.overlay.commit_prospective();
+		let changes = self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache)
+			.expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.backend.commit(
+			changes.transaction_storage_root,
+			changes.transaction,
+		).expect(EXT_NOT_ALLOWED_TO_FAIL);
+		self.storage_transaction_cache.reset();
+	}
 }
 
 impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B>

substrate/primitives/state-machine/src/overlayed_changes.rs

@@ -428,15 +428,18 @@ impl OverlayedChanges {
 	///
 	/// Panics:
 	/// Will panic if there are any uncommitted prospective changes.
-	pub fn into_committed(self) -> (
+	fn drain_committed(&mut self) -> (
 		impl Iterator<Item=(StorageKey, Option<StorageValue>)>,
 		impl Iterator<Item=(StorageKey, (impl Iterator<Item=(StorageKey, Option<StorageValue>)>, OwnedChildInfo))>,
-	){
+	) {
 		assert!(self.prospective.is_empty());
 		(
-			self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
-			self.committed.children.into_iter()
-				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci)))
+			std::mem::replace(&mut self.committed.top, Default::default())
+				.into_iter()
+				.map(|(k, v)| (k, v.value)),
+			std::mem::replace(&mut self.committed.children, Default::default())
+				.into_iter()
+				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))),
 		)
 	}
 
@@ -444,11 +447,22 @@ impl OverlayedChanges {
 	pub fn into_storage_changes<
 		B: Backend<H>, H: Hasher, N: BlockNumber
 	>(
-		self,
+		mut self,
 		backend: &B,
 		changes_trie_state: Option<&ChangesTrieState<H, N>>,
 		parent_hash: H::Out,
 		mut cache: StorageTransactionCache<B::Transaction, H, N>,
+	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
+		self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache)
+	}
+
+	/// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place.
+	pub fn drain_storage_changes<B: Backend<H>, H: Hasher, N: BlockNumber>(
+		&mut self,
+		backend: &B,
+		changes_trie_state: Option<&ChangesTrieState<H, N>>,
+		parent_hash: H::Out,
+		mut cache: &mut StorageTransactionCache<B::Transaction, H, N>,
 	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
 		// If the transaction does not exist, we generate it.
 		if cache.transaction.is_none() {
@@ -474,7 +488,7 @@ impl OverlayedChanges {
 			.take()
 			.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
 
-		let (main_storage_changes, child_storage_changes) = self.into_committed();
+		let (main_storage_changes, child_storage_changes) = self.drain_committed();
 
 		Ok(StorageChanges {
 			main_storage_changes: main_storage_changes.collect(),