Move Externalities into its own crate (#3775)

* Move `Externalities` into `substrate-externalities`

- `Externalities` now support generic extensions
- Split of `primtives-storage` for storage primitive types

* Move the externalities scoping into `substrate-externalities`

* Fix compilation

* Review feedback

* Adds macro for declaring extensions

* Fix benchmarks

* Introduce `ExtensionStore` trait

* Last review comments

* Implement it for `ExtensionStore`

substrate/core/state-machine/src/lib.rs

@@ -18,18 +18,16 @@
 
 #![warn(missing_docs)]
 
-use std::{
-	fmt, result, collections::HashMap,
-	marker::PhantomData, panic::UnwindSafe,
-};
+use std::{fmt, result, collections::HashMap, panic::UnwindSafe, marker::PhantomData};
 use log::{warn, trace};
 use hash_db::Hasher;
 use codec::{Decode, Encode};
 use primitives::{
-	storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain::{self, NeverOffchainExt},
-	traits::{BareCryptoStorePtr, CodeExecutor},
-	hexdisplay::HexDisplay,
+	storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain::OffchainExt,
+	traits::{KeystoreExt, CodeExecutor}, hexdisplay::HexDisplay, hash::H256,
 };
+use overlayed_changes::OverlayedChangeSet;
+use externalities::Extensions;
 
 pub mod backend;
 mod changes_trie;
@@ -42,9 +40,7 @@ mod proving_backend;
 mod trie_backend;
 mod trie_backend_essence;
 
-use overlayed_changes::OverlayedChangeSet;
-pub use trie::{TrieMut, DBValue, MemoryDB};
-pub use trie::trie_types::{Layout, TrieDBMut};
+pub use trie::{trie_types::{Layout, TrieDBMut}, TrieMut, DBValue, MemoryDB};
 pub use testing::TestExternalities;
 pub use basic::BasicExternalities;
 pub use ext::Ext;
@@ -167,48 +163,54 @@ fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E
 }
 
 /// The substrate state machine.
-pub struct StateMachine<'a, B, H, N, T, O, Exec> {
-	backend: B,
-	changes_trie_storage: Option<&'a T>,
-	offchain_ext: Option<&'a mut O>,
-	overlay: &'a mut OverlayedChanges,
+pub struct StateMachine<'a, B, H, N, T, Exec> where H: Hasher<Out=H256>, B: Backend<H> {
+	backend: &'a B,
 	exec: &'a Exec,
 	method: &'a str,
 	call_data: &'a [u8],
-	keystore: Option<BareCryptoStorePtr>,
-	_hasher: PhantomData<(H, N)>,
+	overlay: &'a mut OverlayedChanges,
+	extensions: Extensions,
+	changes_trie_storage: Option<&'a T>,
+	_marker: PhantomData<(H, N)>,
 }
 
-impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
-	H: Hasher,
-	Exec: CodeExecutor<H>,
+impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
+	H: Hasher<Out=H256>,
+	Exec: CodeExecutor,
 	B: Backend<H>,
 	T: ChangesTrieStorage<H, N>,
-	O: offchain::Externalities,
-	H::Out: Ord + 'static,
 	N: crate::changes_trie::BlockNumber,
 {
 	/// Creates new substrate state machine.
 	pub fn new(
-		backend: B,
+		backend: &'a B,
 		changes_trie_storage: Option<&'a T>,
-		offchain_ext: Option<&'a mut O>,
+		offchain_ext: Option<OffchainExt>,
 		overlay: &'a mut OverlayedChanges,
 		exec: &'a Exec,
 		method: &'a str,
 		call_data: &'a [u8],
-		keystore: Option<BareCryptoStorePtr>,
+		keystore: Option<KeystoreExt>,
 	) -> Self {
+		let mut extensions = Extensions::new();
+
+		if let Some(keystore) = keystore {
+			extensions.register(keystore);
+		}
+
+		if let Some(offchain) = offchain_ext {
+			extensions.register(offchain);
+		}
+
 		Self {
 			backend,
-			changes_trie_storage,
-			offchain_ext,
-			overlay,
 			exec,
 			method,
 			call_data,
-			keystore,
-			_hasher: PhantomData,
+			extensions,
+			overlay,
+			changes_trie_storage,
+			_marker: PhantomData,
 		}
 	}
 
@@ -220,10 +222,10 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 	///
 	/// Note: changes to code will be in place if this call is made again. For running partial
 	/// blocks (e.g. a transaction at a time), ensure a different method is used.
-	pub fn execute(
-		&mut self,
-		strategy: ExecutionStrategy,
-	) -> Result<(Vec<u8>, (B::Transaction, H::Out), Option<ChangesTrieTransaction<H, N>>), Box<dyn Error>> {
+	pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<
+		(Vec<u8>, (B::Transaction, H::Out), Option<ChangesTrieTransaction<H, N>>),
+		Box<dyn Error>,
+	> {
 		// We are not giving a native call and thus we are sure that the result can never be a native
 		// value.
 		self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
@@ -252,38 +254,44 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 		R: Decode + Encode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
 	{
-		let mut externalities = ext::Ext::new(
+		let mut ext = Ext::new(
 			self.overlay,
-			&self.backend,
-			self.changes_trie_storage,
-			self.offchain_ext.as_mut().map(|x| &mut **x),
-			self.keystore.clone(),
+			self.backend,
+			self.changes_trie_storage.clone(),
+			Some(&mut self.extensions),
 		);
-		let id = externalities.id;
-		trace!(target: "state-trace", "{:04x}: Call {} at {:?}. Input={:?}",
+
+		let id = ext.id;
+		trace!(
+			target: "state-trace", "{:04x}: Call {} at {:?}. Input={:?}",
 			id,
 			self.method,
 			self.backend,
 			HexDisplay::from(&self.call_data),
 		);
+
 		let (result, was_native) = self.exec.call(
-			&mut externalities,
+			&mut ext,
 			self.method,
 			self.call_data,
 			use_native,
 			native_call,
 		);
+
 		let (storage_delta, changes_delta) = if compute_tx {
-			let (storage_delta, changes_delta) = externalities.transaction();
+			let (storage_delta, changes_delta) = ext.transaction();
 			(Some(storage_delta), changes_delta)
 		} else {
 			(None, None)
 		};
-		trace!(target: "state-trace", "{:04x}: Return. Native={:?}, Result={:?}",
+
+		trace!(
+			target: "state-trace", "{:04x}: Return. Native={:?}, Result={:?}",
 			id,
 			was_native,
 			result,
 		);
+
 		(result, was_native, storage_delta, changes_delta)
 	}
 
@@ -293,12 +301,16 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 		mut native_call: Option<NC>,
 		orig_prospective: OverlayedChangeSet,
 		on_consensus_failure: Handler,
-	) -> (CallResult<R, Exec::Error>, Option<(B::Transaction, H::Out)>, Option<ChangesTrieTransaction<H, N>>) where
+	) -> (
+		CallResult<R, Exec::Error>,
+		Option<(B::Transaction, H::Out)>,
+		Option<ChangesTrieTransaction<H, N>>,
+	) where
 		R: Decode + Encode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
 		Handler: FnOnce(
 			CallResult<R, Exec::Error>,
-			CallResult<R, Exec::Error>
+			CallResult<R, Exec::Error>,
 		) -> CallResult<R, Exec::Error>
 	{
 		let (result, was_native, storage_delta, changes_delta) = self.execute_aux(
@@ -317,7 +329,8 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 
 			if (result.is_ok() && wasm_result.is_ok()
 				&& result.as_ref().ok() == wasm_result.as_ref().ok())
-				|| result.is_err() && wasm_result.is_err() {
+				|| result.is_err() && wasm_result.is_err()
+			{
 				(result, storage_delta, changes_delta)
 			} else {
 				(on_consensus_failure(wasm_result, result), wasm_storage_delta, wasm_changes_delta)
@@ -332,7 +345,11 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 		compute_tx: bool,
 		mut native_call: Option<NC>,
 		orig_prospective: OverlayedChangeSet,
-	) -> (CallResult<R, Exec::Error>, Option<(B::Transaction, H::Out)>, Option<ChangesTrieTransaction<H, N>>) where
+	) -> (
+		CallResult<R, Exec::Error>,
+		Option<(B::Transaction, H::Out)>,
+		Option<ChangesTrieTransaction<H, N>>,
+	) where
 		R: Decode + Encode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
 	{
@@ -431,7 +448,7 @@ impl<'a, B, H, N, T, O, Exec> StateMachine<'a, B, H, N, T, O, Exec> where
 		};
 
 		if result.is_ok() {
-			init_overlay(self.overlay, true, &self.backend)?;
+			init_overlay(self.overlay, true, self.backend)?;
 		}
 
 		result.map_err(|e| Box::new(e) as _)
@@ -445,13 +462,12 @@ pub fn prove_execution<B, H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-	keystore: Option<BareCryptoStorePtr>,
+	keystore: Option<KeystoreExt>,
 ) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<dyn Error>>
 where
 	B: Backend<H>,
-	H: Hasher,
-	Exec: CodeExecutor<H>,
-	H::Out: Ord + 'static,
+	H: Hasher<Out=H256>,
+	Exec: CodeExecutor,
 {
 	let trie_backend = backend.as_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
@@ -473,17 +489,16 @@ pub fn prove_execution_on_trie_backend<S, H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-	keystore: Option<BareCryptoStorePtr>,
+	keystore: Option<KeystoreExt>,
 ) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<dyn Error>>
 where
 	S: trie_backend_essence::TrieBackendStorage<H>,
-	H: Hasher,
-	Exec: CodeExecutor<H>,
-	H::Out: Ord + 'static,
+	H: Hasher<Out=H256>,
+	Exec: CodeExecutor,
 {
 	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
-	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, _, Exec>::new(
-		proving_backend, None, NeverOffchainExt::new(), overlay, exec, method, call_data, keystore,
+	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
+		&proving_backend, None, None, overlay, exec, method, call_data, keystore,
 	);
 
 	let (result, _, _) = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
@@ -503,11 +518,11 @@ pub fn execution_proof_check<H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-	keystore: Option<BareCryptoStorePtr>,
+	keystore: Option<KeystoreExt>,
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
-	H: Hasher,
-	Exec: CodeExecutor<H>,
+	H: Hasher<Out=H256>,
+	Exec: CodeExecutor,
 	H::Out: Ord + 'static,
 {
 	let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
@@ -521,15 +536,14 @@ pub fn execution_proof_check_on_trie_backend<H, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-	keystore: Option<BareCryptoStorePtr>,
+	keystore: Option<KeystoreExt>,
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
-	H: Hasher,
-	Exec: CodeExecutor<H>,
-	H::Out: Ord + 'static,
+	H: Hasher<Out=H256>,
+	Exec: CodeExecutor,
 {
-	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, _, Exec>::new(
-		trie_backend, None, NeverOffchainExt::new(), overlay, exec, method, call_data, keystore,
+	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
+		trie_backend, None, None, overlay, exec, method, call_data, keystore,
 	);
 
 	sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
@@ -546,7 +560,7 @@ pub fn prove_read<B, H, I>(
 ) -> Result<Vec<Vec<u8>>, Box<dyn Error>>
 where
 	B: Backend<H>,
-	H: Hasher,
+	H: Hasher<Out=H256>,
 	H::Out: Ord,
 	I: IntoIterator,
 	I::Item: AsRef<[u8]>,
@@ -741,7 +755,7 @@ mod tests {
 		InMemoryStorage as InMemoryChangesTrieStorage,
 		Configuration as ChangesTrieConfig,
 	};
-	use primitives::{Blake2Hasher, map, traits::Externalities, child_storage_key::ChildStorageKey};
+	use primitives::{Blake2Hasher, map, traits::Externalities, storage::ChildStorageKey};
 
 	struct DummyCodeExecutor {
 		change_changes_trie_config: bool,
@@ -750,10 +764,14 @@ mod tests {
 		fallback_succeeds: bool,
 	}
 
-	impl<H: Hasher> CodeExecutor<H> for DummyCodeExecutor {
+	impl CodeExecutor for DummyCodeExecutor {
 		type Error = u8;
 
-		fn call<E: Externalities<H>, R: Encode + Decode + PartialEq, NC: FnOnce() -> result::Result<R, String>>(
+		fn call<
+			E: Externalities,
+			R: Encode + Decode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String>,
+		>(
 			&self,
 			ext: &mut E,
 			_method: &str,
@@ -800,9 +818,9 @@ mod tests {
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 
 		let mut state_machine = StateMachine::new(
-			backend,
+			&backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -829,9 +847,9 @@ mod tests {
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 
 		let mut state_machine = StateMachine::new(
-			backend,
+			&backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -855,9 +873,9 @@ mod tests {
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 
 		let mut state_machine = StateMachine::new(
-			backend,
+			&backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -948,7 +966,6 @@ mod tests {
 				&mut overlay,
 				backend,
 				Some(&changes_trie_storage),
-				NeverOffchainExt::new(),
 				None,
 			);
 			ext.clear_prefix(b"ab");
@@ -979,7 +996,6 @@ mod tests {
 			&mut overlay,
 			backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
 			None,
 		);
 
@@ -1067,9 +1083,9 @@ mod tests {
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 
 		let mut state_machine = StateMachine::new(
-			backend,
+			&backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: true,
@@ -1092,9 +1108,9 @@ mod tests {
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 
 		let mut state_machine = StateMachine::new(
-			backend,
+			&backend,
 			Some(&changes_trie_storage),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: true,