Move Externalities into its own crate (#3775)

* Move `Externalities` into `substrate-externalities`

- `Externalities` now support generic extensions
- Split of `primtives-storage` for storage primitive types

* Move the externalities scoping into `substrate-externalities`

* Fix compilation

* Review feedback

* Adds macro for declaring extensions

* Fix benchmarks

* Introduce `ExtensionStore` trait

* Last review comments

* Implement it for `ExtensionStore`

substrate/core/executor/src/lib.rs

@@ -45,7 +45,7 @@ pub use native_executor::{with_native_environment, NativeExecutor, NativeExecuti
 pub use runtime_version::{RuntimeVersion, NativeVersion};
 pub use codec::Codec;
 #[doc(hidden)]
-pub use primitives::{Blake2Hasher, traits::Externalities};
+pub use primitives::traits::Externalities;
 #[doc(hidden)]
 pub use wasm_interface;
 pub use wasm_runtime::WasmExecutionMethod;
@@ -56,7 +56,7 @@ pub trait RuntimeInfo {
 	fn native_version(&self) -> &NativeVersion;
 
 	/// Extract RuntimeVersion of given :code block
-	fn runtime_version<E: Externalities<Blake2Hasher>> (
+	fn runtime_version<E: Externalities> (
 		&self,
 		ext: &mut E,
 	) -> Option<RuntimeVersion>;

substrate/core/executor/src/native_executor.rs

@@ -20,7 +20,7 @@ use crate::wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime};
 use crate::RuntimeInfo;
 use runtime_version::{NativeVersion, RuntimeVersion};
 use codec::{Decode, Encode};
-use primitives::{Blake2Hasher, NativeOrEncoded, traits::{CodeExecutor, Externalities}};
+use primitives::{NativeOrEncoded, traits::{CodeExecutor, Externalities}};
 use log::{trace, warn};
 
 thread_local! {
@@ -41,10 +41,10 @@ fn safe_call<F, U>(f: F) -> Result<U>
 /// Set up the externalities and safe calling environment to execute calls to a native runtime.
 ///
 /// If the inner closure panics, it will be caught and return an error.
-pub fn with_native_environment<F, U>(ext: &mut dyn Externalities<Blake2Hasher>, f: F) -> Result<U>
+pub fn with_native_environment<F, U>(ext: &mut dyn Externalities, f: F) -> Result<U>
 	where F: UnwindSafe + FnOnce() -> U
 {
-	runtime_io::with_externalities(ext, move || safe_call(f))
+	externalities::set_and_run_with_externalities(ext, move || safe_call(f))
 }
 
 /// Delegate for dispatching a CodeExecutor call.
@@ -54,7 +54,7 @@ pub trait NativeExecutionDispatch: Send + Sync {
 	/// Dispatch a method in the runtime.
 	///
 	/// If the method with the specified name doesn't exist then `Err` is returned.
-	fn dispatch(ext: &mut dyn Externalities<Blake2Hasher>, method: &str, data: &[u8]) -> Result<Vec<u8>>;
+	fn dispatch(ext: &mut dyn Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>>;
 
 	/// Provide native runtime version.
 	fn native_version() -> NativeVersion;
@@ -95,10 +95,8 @@ impl<D: NativeExecutionDispatch> NativeExecutor<D> {
 	fn with_runtime<E, R>(
 		&self,
 		ext: &mut E,
-		f: impl for <'a> FnOnce(&'a mut dyn WasmRuntime, &'a mut E) -> Result<R>
-	) -> Result<R>
-		where E: Externalities<Blake2Hasher>
-	{
+		f: impl for <'a> FnOnce(&'a mut dyn WasmRuntime, &'a mut E) -> Result<R>,
+	) -> Result<R> where E: Externalities {
 		RUNTIMES_CACHE.with(|cache| {
 			let mut cache = cache.borrow_mut();
 			let runtime = cache.fetch_runtime(ext, self.fallback_method, self.default_heap_pages)?;
@@ -123,7 +121,7 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
 		&self.native_version
 	}
 
-	fn runtime_version<E: Externalities<Blake2Hasher>>(
+	fn runtime_version<E: Externalities>(
 		&self,
 		ext: &mut E,
 	) -> Option<RuntimeVersion> {
@@ -137,12 +135,12 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
 	}
 }
 
-impl<D: NativeExecutionDispatch> CodeExecutor<Blake2Hasher> for NativeExecutor<D> {
+impl<D: NativeExecutionDispatch> CodeExecutor for NativeExecutor<D> {
 	type Error = Error;
 
 	fn call
 	<
-		E: Externalities<Blake2Hasher>,
+		E: Externalities,
 		R:Decode + Encode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe
 	>(
@@ -220,7 +218,7 @@ macro_rules! native_executor_instance {
 	(IMPL $name:ident, $dispatcher:path, $version:path) => {
 		impl $crate::NativeExecutionDispatch for $name {
 			fn dispatch(
-				ext: &mut $crate::Externalities<$crate::Blake2Hasher>,
+				ext: &mut $crate::Externalities,
 				method: &str,
 				data: &[u8]
 			) -> $crate::error::Result<Vec<u8>> {

substrate/core/executor/src/sandbox.rs

@@ -593,9 +593,9 @@ mod tests {
 	use wabt;
 	use runtime_test::WASM_BINARY;
 
-	type TestExternalities<H> = CoreTestExternalities<H, u64>;
+	type TestExternalities = CoreTestExternalities<Blake2Hasher, u64>;
 
-	fn call_wasm<E: Externalities<Blake2Hasher>>(
+	fn call_wasm<E: Externalities>(
 		ext: &mut E,
 		heap_pages: u64,
 		code: &[u8],
@@ -609,7 +609,7 @@ mod tests {
 
 	#[test]
 	fn sandbox_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -641,7 +641,7 @@ mod tests {
 
 	#[test]
 	fn sandbox_trap() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -662,7 +662,7 @@ mod tests {
 
 	#[test]
 	fn sandbox_should_trap_when_heap_exhausted() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -690,7 +690,7 @@ mod tests {
 
 	#[test]
 	fn start_called() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -728,7 +728,7 @@ mod tests {
 
 	#[test]
 	fn invoke_args() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -762,7 +762,7 @@ mod tests {
 
 	#[test]
 	fn return_val() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -784,7 +784,7 @@ mod tests {
 
 	#[test]
 	fn unlinkable_module() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -804,7 +804,7 @@ mod tests {
 
 	#[test]
 	fn corrupted_module() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		// Corrupted wasm file
@@ -818,7 +818,7 @@ mod tests {
 
 	#[test]
 	fn start_fn_ok() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"
@@ -841,7 +841,7 @@ mod tests {
 
 	#[test]
 	fn start_fn_traps() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 
 		let code = wabt::wat2wasm(r#"

substrate/core/executor/src/wasm_runtime.rs

@@ -23,7 +23,7 @@ use crate::error::{Error, WasmError};
 use crate::wasmi_execution;
 use log::{trace, warn};
 use codec::Decode;
-use primitives::{storage::well_known_keys, Blake2Hasher, traits::Externalities};
+use primitives::{storage::well_known_keys, traits::Externalities};
 use runtime_version::RuntimeVersion;
 use std::{collections::hash_map::{Entry, HashMap}};
 
@@ -36,8 +36,8 @@ pub trait WasmRuntime {
 	fn update_heap_pages(&mut self, heap_pages: u64) -> bool;
 
 	/// Call a method in the Substrate runtime by name. Returns the encoded result on success.
-	fn call(&mut self, ext: &mut dyn Externalities<Blake2Hasher>, method: &str, data: &[u8])
-			-> Result<Vec<u8>, Error>;
+	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8])
+		-> Result<Vec<u8>, Error>;
 
 	/// Returns the version of this runtime.
 	///
@@ -109,7 +109,7 @@ impl RuntimesCache {
 	///
 	/// `Error::InvalidMemoryReference` is returned if no memory export with the
 	/// identifier `memory` can be found in the runtime.
-	pub fn fetch_runtime<E: Externalities<Blake2Hasher>>(
+	pub fn fetch_runtime<E: Externalities>(
 		&mut self,
 		ext: &mut E,
 		wasm_method: WasmExecutionMethod,
@@ -157,7 +157,7 @@ impl RuntimesCache {
 	}
 }
 
-fn create_wasm_runtime<E: Externalities<Blake2Hasher>>(
+fn create_wasm_runtime<E: Externalities>(
 	ext: &mut E,
 	wasm_method: WasmExecutionMethod,
 	heap_pages: u64,

substrate/core/executor/src/wasmi_execution.rs

@@ -23,7 +23,7 @@ use wasmi::{
 };
 use crate::error::{Error, WasmError};
 use codec::{Encode, Decode};
-use primitives::{sandbox as sandbox_primitives, Blake2Hasher, traits::Externalities};
+use primitives::{sandbox as sandbox_primitives, traits::Externalities};
 use crate::host_interface::SubstrateExternals;
 use crate::sandbox;
 use crate::allocator;
@@ -342,7 +342,7 @@ fn get_heap_base(module: &ModuleRef) -> Result<u32, Error> {
 
 /// Call a given method in the given wasm-module runtime.
 fn call_in_wasm_module(
-	ext: &mut dyn Externalities<Blake2Hasher>,
+	ext: &mut dyn Externalities,
 	module_instance: &ModuleRef,
 	method: &str,
 	data: &[u8],
@@ -373,7 +373,7 @@ fn call_in_wasm_module_with_custom_signature<
 	FR: FnOnce(Option<RuntimeValue>, &MemoryRef) -> Result<Option<R>, Error>,
 	R,
 >(
-	ext: &mut dyn Externalities<Blake2Hasher>,
+	ext: &mut dyn Externalities,
 	module_instance: &ModuleRef,
 	method: &str,
 	create_parameters: F,
@@ -398,7 +398,7 @@ fn call_in_wasm_module_with_custom_signature<
 		fec.write_memory(offset, data).map(|_| offset.into()).map_err(Into::into)
 	})?;
 
-	let result = runtime_io::with_externalities(
+	let result = externalities::set_and_run_with_externalities(
 		ext,
 		|| module_instance.invoke_export(method, &parameters, &mut fec),
 	);
@@ -588,7 +588,7 @@ impl WasmRuntime for WasmiRuntime {
 		self.state_snapshot.heap_pages == heap_pages
 	}
 
-	fn call(&mut self, ext: &mut dyn Externalities<Blake2Hasher>, method: &str, data: &[u8])
+	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8])
 			-> Result<Vec<u8>, Error>
 	{
 		self.with(|module| {
@@ -601,7 +601,7 @@ impl WasmRuntime for WasmiRuntime {
 	}
 }
 
-pub fn create_instance<E: Externalities<Blake2Hasher>>(ext: &mut E, code: &[u8], heap_pages: u64)
+pub fn create_instance<E: Externalities>(ext: &mut E, code: &[u8], heap_pages: u64)
 	-> Result<WasmiRuntime, WasmError>
 {
 	let module = Module::from_buffer(&code).map_err(|_| WasmError::InvalidModule)?;
@@ -656,14 +656,16 @@ mod tests {
 
 	use state_machine::TestExternalities as CoreTestExternalities;
 	use hex_literal::hex;
-	use primitives::{blake2_128, blake2_256, ed25519, sr25519, map, Pair};
+	use primitives::{
+		Blake2Hasher, blake2_128, blake2_256, ed25519, sr25519, map, Pair, offchain::OffchainExt,
+	};
 	use runtime_test::WASM_BINARY;
 	use substrate_offchain::testing;
 	use trie::{TrieConfiguration, trie_types::Layout};
 
-	type TestExternalities<H> = CoreTestExternalities<H, u64>;
+	type TestExternalities = CoreTestExternalities<Blake2Hasher, u64>;
 
-	fn call<E: Externalities<Blake2Hasher>>(
+	fn call<E: Externalities>(
 		ext: &mut E,
 		heap_pages: u64,
 		code: &[u8],
@@ -798,7 +800,7 @@ mod tests {
 
 	#[test]
 	fn ed25519_verify_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 		let key = ed25519::Pair::from_seed(&blake2_256(b"test"));
 		let sig = key.sign(b"all ok!");
@@ -824,7 +826,7 @@ mod tests {
 
 	#[test]
 	fn sr25519_verify_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let test_code = WASM_BINARY;
 		let key = sr25519::Pair::from_seed(&blake2_256(b"test"));
 		let sig = key.sign(b"all ok!");
@@ -850,7 +852,7 @@ mod tests {
 
 	#[test]
 	fn ordered_trie_root_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()];
 		let test_code = WASM_BINARY;
 		assert_eq!(
@@ -863,9 +865,9 @@ mod tests {
 	fn offchain_local_storage_should_work() {
 		use substrate_client::backend::OffchainStorage;
 
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let (offchain, state) = testing::TestOffchainExt::new();
-		ext.set_offchain_externalities(offchain);
+		ext.register_extension(OffchainExt::new(offchain));
 		let test_code = WASM_BINARY;
 		assert_eq!(
 			call(&mut ext, 8, &test_code[..], "test_offchain_local_storage", &[]).unwrap(),
@@ -876,9 +878,9 @@ mod tests {
 
 	#[test]
 	fn offchain_http_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::default();
+		let mut ext = TestExternalities::default();
 		let (offchain, state) = testing::TestOffchainExt::new();
-		ext.set_offchain_externalities(offchain);
+		ext.register_extension(OffchainExt::new(offchain));
 		state.write().expect_request(
 			0,
 			testing::PendingRequest {