Move Externalities into its own crate (#3775)

* Move `Externalities` into `substrate-externalities`

- `Externalities` now support generic extensions
- Split of `primtives-storage` for storage primitive types

* Move the externalities scoping into `substrate-externalities`

* Fix compilation

* Review feedback

* Adds macro for declaring extensions

* Fix benchmarks

* Introduce `ExtensionStore` trait

* Last review comments

* Implement it for `ExtensionStore`

substrate/core/client/src/call_executor.rs

@@ -26,8 +26,8 @@ use state_machine::{
 use executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use hash_db::Hasher;
 use primitives::{
-	offchain::{self, NeverOffchainExt}, H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue,
-	traits::CodeExecutor,
+	offchain::OffchainExt, H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue,
+	traits::{CodeExecutor, KeystoreExt},
 };
 
 use crate::runtime_api::{ProofRecorder, InitializeBlock};
@@ -47,15 +47,13 @@ where
 	/// Execute a call to a contract on top of state in a block of given hash.
 	///
 	/// No changes are made.
-	fn call<
-		O: offchain::Externalities,
-	>(
+	fn call(
 		&self,
 		id: &BlockId<B>,
 		method: &str,
 		call_data: &[u8],
 		strategy: ExecutionStrategy,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 	) -> Result<Vec<u8>, error::Error>;
 
 	/// Execute a contextual call on top of state in a block of a given hash.
@@ -65,7 +63,6 @@ where
 	/// of the execution context.
 	fn contextual_call<
 		'a,
-		O: offchain::Externalities,
 		IB: Fn() -> error::Result<()>,
 		EM: Fn(
 			Result<NativeOrEncoded<R>, Self::Error>,
@@ -83,7 +80,7 @@ where
 		initialize_block: InitializeBlock<'a, B>,
 		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 		proof_recorder: &Option<Rc<RefCell<ProofRecorder<B>>>>,
 		enable_keystore: bool,
 	) -> error::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
@@ -97,11 +94,10 @@ where
 	///
 	/// No changes are made.
 	fn call_at_state<
-		O: offchain::Externalities,
 		S: state_machine::Backend<H>,
 		F: FnOnce(
 			Result<NativeOrEncoded<R>, Self::Error>,
-			Result<NativeOrEncoded<R>, Self::Error>
+			Result<NativeOrEncoded<R>, Self::Error>,
 		) -> Result<NativeOrEncoded<R>, Self::Error>,
 		R: Encode + Decode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
@@ -112,7 +108,7 @@ where
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
 		native_call: Option<NC>,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 	) -> Result<
 		(
 			NativeOrEncoded<R>,
@@ -191,18 +187,18 @@ impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
 impl<B, E, Block> CallExecutor<Block, Blake2Hasher> for LocalCallExecutor<B, E>
 where
 	B: backend::Backend<Block, Blake2Hasher>,
-	E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+	E: CodeExecutor + RuntimeInfo,
 	Block: BlockT<Hash=H256>,
 {
 	type Error = E::Error;
 
-	fn call<O: offchain::Externalities>(
+	fn call(
 		&self,
 		id: &BlockId<Block>,
 		method: &str,
 		call_data: &[u8],
 		strategy: ExecutionStrategy,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 	) -> error::Result<Vec<u8>> {
 		let mut changes = OverlayedChanges::default();
 		let state = self.backend.state_at(*id)?;
@@ -214,7 +210,7 @@ where
 			&self.executor,
 			method,
 			call_data,
-			self.keystore.clone(),
+			self.keystore.clone().map(KeystoreExt),
 		).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 			strategy.get_manager(),
 			false,
@@ -227,7 +223,6 @@ where
 
 	fn contextual_call<
 		'a,
-		O: offchain::Externalities,
 		IB: Fn() -> error::Result<()>,
 		EM: Fn(
 			Result<NativeOrEncoded<R>, Self::Error>,
@@ -245,7 +240,7 @@ where
 		initialize_block: InitializeBlock<'a, Block>,
 		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
 		enable_keystore: bool,
 	) -> Result<NativeOrEncoded<R>, error::Error> where ExecutionManager<EM>: Clone {
@@ -259,7 +254,7 @@ where
 		}
 
 		let keystore = if enable_keystore {
-			self.keystore.clone()
+			self.keystore.clone().map(KeystoreExt)
 		} else {
 			None
 		};
@@ -326,7 +321,6 @@ where
 			&mut overlay,
 			&state,
 			self.backend.changes_trie_storage(),
-			NeverOffchainExt::new(),
 			None,
 		);
 		let version = self.executor.runtime_version(&mut ext);
@@ -335,11 +329,10 @@ where
 	}
 
 	fn call_at_state<
-		O: offchain::Externalities,
 		S: state_machine::Backend<Blake2Hasher>,
 		F: FnOnce(
 			Result<NativeOrEncoded<R>, Self::Error>,
-			Result<NativeOrEncoded<R>, Self::Error>
+			Result<NativeOrEncoded<R>, Self::Error>,
 		) -> Result<NativeOrEncoded<R>, Self::Error>,
 		R: Encode + Decode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
@@ -350,7 +343,7 @@ where
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
 		native_call: Option<NC>,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 	) -> error::Result<(
 		NativeOrEncoded<R>,
 		(S::Transaction, <Blake2Hasher as Hasher>::Out),
@@ -364,7 +357,7 @@ where
 			&self.executor,
 			method,
 			call_data,
-			self.keystore.clone(),
+			self.keystore.clone().map(KeystoreExt),
 		).execute_using_consensus_failure_handler(
 			manager,
 			true,
@@ -391,7 +384,7 @@ where
 			&self.executor,
 			method,
 			call_data,
-			self.keystore.clone(),
+			self.keystore.clone().map(KeystoreExt),
 		)
 		.map_err(Into::into)
 	}

substrate/core/client/src/client.rs

@@ -28,7 +28,7 @@ use hash_db::{Hasher, Prefix};
 use primitives::{
 	Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, NeverNativeValue, ExecutionContext,
 	NativeOrEncoded, storage::{StorageKey, StorageData, well_known_keys},
-	offchain::{NeverOffchainExt, self}, traits::CodeExecutor,
+	offchain::{OffchainExt, self}, traits::CodeExecutor,
 };
 use substrate_telemetry::{telemetry, SUBSTRATE_INFO};
 use sr_primitives::{
@@ -248,7 +248,7 @@ pub fn new_in_mem<E, Block, S, RA>(
 	Block,
 	RA
 >> where
-	E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+	E: CodeExecutor + RuntimeInfo,
 	S: BuildStorage,
 	Block: BlockT<Hash=H256>,
 {
@@ -264,7 +264,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 	keystore: Option<primitives::traits::BareCryptoStorePtr>,
 ) -> error::Result<Client<B, LocalCallExecutor<B, E>, Block, RA>>
 	where
-		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+		E: CodeExecutor + RuntimeInfo,
 		S: BuildStorage,
 		Block: BlockT<Hash=H256>,
 		B: backend::LocalBackend<Block, Blake2Hasher>
@@ -1058,18 +1058,27 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 						}),
 					}
 				};
-				let (_, storage_update, changes_update) = self.executor.call_at_state::<_, _, _, NeverNativeValue, fn() -> _>(
-					transaction_state,
-					&mut overlay,
-					"Core_execute_block",
-					&<Block as BlockT>::new(import_headers.pre().clone(), body.unwrap_or_default()).encode(),
-					match origin {
-						BlockOrigin::NetworkInitialSync => get_execution_manager(self.execution_strategies().syncing),
-						_ => get_execution_manager(self.execution_strategies().importing),
-					},
-					None,
-					NeverOffchainExt::new(),
-				)?;
+
+				let encoded_block = <Block as BlockT>::new(
+					import_headers.pre().clone(),
+					body.unwrap_or_default(),
+				).encode();
+
+				let (_, storage_update, changes_update) = self.executor
+					.call_at_state::<_, _, NeverNativeValue, fn() -> _>(
+						transaction_state,
+						&mut overlay,
+						"Core_execute_block",
+						&encoded_block,
+						match origin {
+							BlockOrigin::NetworkInitialSync => get_execution_manager(
+								self.execution_strategies().syncing,
+							),
+							_ => get_execution_manager(self.execution_strategies().importing),
+						},
+						None,
+						None,
+					)?;
 
 				overlay.commit_prospective();
 
@@ -1460,12 +1469,13 @@ impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 		};
 
 		let capabilities = context.capabilities();
-		let mut offchain_extensions = match context {
-			ExecutionContext::OffchainCall(ext) => ext.map(|x| x.0),
-			_ => None,
-		}.map(|ext| offchain::LimitedExternalities::new(capabilities, ext));
+		let offchain_extensions = if let ExecutionContext::OffchainCall(Some(ext)) = context {
+			Some(OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)))
+		} else {
+			None
+		};
 
-		self.executor.contextual_call::<_, _, fn(_,_) -> _,_,_>(
+		self.executor.contextual_call::<_, fn(_,_) -> _,_,_>(
 			|| core_api.initialize_block(at, &self.prepare_environment_block(at)?),
 			at,
 			function,
@@ -1474,7 +1484,7 @@ impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 			initialize_block,
 			manager,
 			native_call,
-			offchain_extensions.as_mut(),
+			offchain_extensions,
 			recorder,
 			capabilities.has(offchain::Capability::Keystore),
 		)

substrate/core/client/src/genesis.rs

@@ -53,7 +53,7 @@ mod tests {
 		runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest},
 		AccountKeyring, Sr25519Keyring,
 	};
-	use primitives::{Blake2Hasher, map, offchain::NeverOffchainExt};
+	use primitives::{Blake2Hasher, map};
 	use hex_literal::*;
 
 	native_executor_instance!(
@@ -93,7 +93,7 @@ mod tests {
 		StateMachine::new(
 			backend,
 			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlay,
 			&executor(),
 			"Core_initialize_block",
@@ -107,7 +107,7 @@ mod tests {
 			StateMachine::new(
 				backend,
 				Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-				NeverOffchainExt::new(),
+				None,
 				&mut overlay,
 				&executor(),
 				"BlockBuilder_apply_extrinsic",
@@ -121,7 +121,7 @@ mod tests {
 		let (ret_data, _, _) = StateMachine::new(
 			backend,
 			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlay,
 			&executor(),
 			"BlockBuilder_finalize_block",
@@ -169,7 +169,7 @@ mod tests {
 		let _ = StateMachine::new(
 			&backend,
 			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlay,
 			&executor(),
 			"Core_execute_block",
@@ -199,7 +199,7 @@ mod tests {
 		let _ = StateMachine::new(
 			&backend,
 			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlay,
 			&executor(),
 			"Core_execute_block",
@@ -229,7 +229,7 @@ mod tests {
 		let r = StateMachine::new(
 			&backend,
 			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
-			NeverOffchainExt::new(),
+			None,
 			&mut overlay,
 			&executor(),
 			"Core_execute_block",

substrate/core/client/src/light/call_executor.rs

@@ -23,11 +23,12 @@ use std::{
 
 use codec::{Encode, Decode};
 use primitives::{
-	offchain, H256, Blake2Hasher, convert_hash, NativeOrEncoded,
+	offchain::OffchainExt, H256, Blake2Hasher, convert_hash, NativeOrEncoded,
 	traits::CodeExecutor,
 };
-use sr_primitives::generic::BlockId;
-use sr_primitives::traits::{One, Block as BlockT, Header as HeaderT, NumberFor};
+use sr_primitives::{
+	generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, NumberFor},
+};
 use state_machine::{
 	self, Backend as StateBackend, OverlayedChanges, ExecutionStrategy, create_proof_check_backend,
 	execution_proof_check_on_trie_backend, ExecutionManager, ChangesTrieTransaction,
@@ -74,15 +75,13 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 {
 	type Error = ClientError;
 
-	fn call<
-		O: offchain::Externalities,
-	>(
+	fn call(
 		&self,
 		id: &BlockId<Block>,
 		method: &str,
 		call_data: &[u8],
 		strategy: ExecutionStrategy,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 	) -> ClientResult<Vec<u8>> {
 		match self.backend.is_local_state_available(id) {
 			true => self.local.call(id, method, call_data, strategy, side_effects_handler),
@@ -92,7 +91,6 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 
 	fn contextual_call<
 		'a,
-		O: offchain::Externalities,
 		IB: Fn() -> ClientResult<()>,
 		EM: Fn(
 			Result<NativeOrEncoded<R>, Self::Error>,
@@ -110,7 +108,7 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 		initialize_block: InitializeBlock<'a, Block>,
 		_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
-		side_effects_handler: Option<&mut O>,
+		side_effects_handler: Option<OffchainExt>,
 		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
 		enable_keystore: bool,
 	) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
@@ -119,7 +117,6 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 
 		match self.backend.is_local_state_available(at) {
 			true => CallExecutor::contextual_call::<
-				_,
 				_,
 				fn(
 					Result<NativeOrEncoded<R>, Local::Error>,
@@ -153,7 +150,6 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 	}
 
 	fn call_at_state<
-		O: offchain::Externalities,
 		S: StateBackend<Blake2Hasher>,
 		FF: FnOnce(
 			Result<NativeOrEncoded<R>, Self::Error>,
@@ -168,7 +164,7 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 		_call_data: &[u8],
 		_manager: ExecutionManager<FF>,
 		_native_call: Option<NC>,
-		_side_effects_handler: Option<&mut O>,
+		_side_effects_handler: Option<OffchainExt>,
 	) -> ClientResult<(
 		NativeOrEncoded<R>,
 		(S::Transaction, <Blake2Hasher as Hasher>::Out),
@@ -242,11 +238,10 @@ pub fn check_execution_proof<Header, E, H>(
 ) -> ClientResult<Vec<u8>>
 	where
 		Header: HeaderT,
-		E: CodeExecutor<H>,
-		H: Hasher,
-		H::Out: Ord + 'static,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
 {
-	check_execution_proof_with_make_header(
+	check_execution_proof_with_make_header::<Header, E, H, _>(
 		executor,
 		request,
 		remote_proof,
@@ -268,9 +263,8 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
 ) -> ClientResult<Vec<u8>>
 	where
 		Header: HeaderT,
-		E: CodeExecutor<H>,
-		H: Hasher,
-		H::Out: Ord + 'static,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
 {
 	let local_state_root = request.header.state_root();
 	let root: H::Out = convert_hash(&local_state_root);
@@ -301,33 +295,32 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
 
 #[cfg(test)]
 mod tests {
+	use super::*;
 	use consensus::BlockOrigin;
-	use primitives::offchain::NeverOffchainExt;
 	use test_client::{self, runtime::{Header, Digest, Block}, ClientExt, TestClient};
 	use executor::{NativeExecutor, WasmExecutionMethod};
+	use primitives::Blake2Hasher;
 	use crate::backend::{Backend, NewBlockState};
 	use crate::in_mem::Backend as InMemBackend;
-	use super::*;
 
 	struct DummyCallExecutor;
 
 	impl CallExecutor<Block, Blake2Hasher> for DummyCallExecutor {
 		type Error = ClientError;
 
-		fn call<O: offchain::Externalities>(
+		fn call(
 			&self,
 			_id: &BlockId<Block>,
 			_method: &str,
 			_call_data: &[u8],
 			_strategy: ExecutionStrategy,
-			_side_effects_handler: Option<&mut O>,
+			_side_effects_handler: Option<OffchainExt>,
 		) -> Result<Vec<u8>, ClientError> {
 			Ok(vec![42])
 		}
 
 		fn contextual_call<
 			'a,
-			O: offchain::Externalities,
 			IB: Fn() -> ClientResult<()>,
 			EM: Fn(
 				Result<NativeOrEncoded<R>, Self::Error>,
@@ -345,7 +338,7 @@ mod tests {
 			_initialize_block: InitializeBlock<'a, Block>,
 			_execution_manager: ExecutionManager<EM>,
 			_native_call: Option<NC>,
-			_side_effects_handler: Option<&mut O>,
+			_side_effects_handler: Option<OffchainExt>,
 			_proof_recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
 			_enable_keystore: bool,
 		) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
@@ -357,7 +350,6 @@ mod tests {
 		}
 
 		fn call_at_state<
-			O: offchain::Externalities,
 			S: state_machine::Backend<Blake2Hasher>,
 			F: FnOnce(
 				Result<NativeOrEncoded<R>, Self::Error>,
@@ -372,7 +364,7 @@ mod tests {
 			_call_data: &[u8],
 			_manager: ExecutionManager<F>,
 			_native_call: Option<NC>,
-			_side_effects_handler: Option<&mut O>,
+			_side_effects_handler: Option<OffchainExt>,
 		) -> Result<
 			(
 				NativeOrEncoded<R>,
@@ -417,13 +409,17 @@ mod tests {
 			).unwrap();
 
 			// check remote execution proof locally
-			let local_result = check_execution_proof(&local_executor(), &RemoteCallRequest {
-				block: test_client::runtime::Hash::default(),
-				header: remote_header,
-				method: method.into(),
-				call_data: vec![],
-				retry_count: None,
-			}, remote_execution_proof).unwrap();
+			let local_result = check_execution_proof::<_, _, Blake2Hasher>(
+				&local_executor(),
+				&RemoteCallRequest {
+					block: test_client::runtime::Hash::default(),
+					header: remote_header,
+					method: method.into(),
+					call_data: vec![],
+					retry_count: None,
+				},
+				remote_execution_proof,
+			).unwrap();
 
 			(remote_result, local_result)
 		}
@@ -440,7 +436,7 @@ mod tests {
 			).unwrap();
 
 			// check remote execution proof locally
-			let execution_result = check_execution_proof_with_make_header(
+			let execution_result = check_execution_proof_with_make_header::<_, _, Blake2Hasher, _>(
 				&local_executor(),
 				&RemoteCallRequest {
 					block: test_client::runtime::Hash::default(),
@@ -518,7 +514,7 @@ mod tests {
 				"test_method",
 				&[],
 				ExecutionStrategy::NativeElseWasm,
-				NeverOffchainExt::new(),
+				None,
 			).unwrap(),
 			vec![42],
 		);
@@ -528,7 +524,7 @@ mod tests {
 			"test_method",
 			&[],
 			ExecutionStrategy::NativeElseWasm,
-			NeverOffchainExt::new(),
+			None,
 		);
 
 		match call_on_unavailable {

substrate/core/client/src/light/fetcher.rs

@@ -23,7 +23,7 @@ use std::future::Future;
 
 use hash_db::{HashDB, Hasher, EMPTY_PREFIX};
 use codec::{Decode, Encode};
-use primitives::{ChangesTrieConfiguration, convert_hash, traits::CodeExecutor};
+use primitives::{ChangesTrieConfiguration, convert_hash, traits::CodeExecutor, H256};
 use sr_primitives::traits::{
 	Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor,
 	SimpleArithmetic, CheckedConversion, Zero,
@@ -370,9 +370,8 @@ impl<E, H, B: BlockT, S: BlockchainStorage<B>> LightDataChecker<E, H, B, S> {
 impl<E, Block, H, S> FetchChecker<Block> for LightDataChecker<E, H, Block, S>
 	where
 		Block: BlockT,
-		E: CodeExecutor<H>,
-		H: Hasher,
-		H::Out: Ord + 'static,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
 		S: BlockchainStorage<Block>,
 {
 	fn check_header_proof(

substrate/core/client/src/light/mod.rs

@@ -63,7 +63,7 @@ pub fn new_light<B, S, GS, RA, E>(
 		B: BlockT<Hash=H256>,
 		S: BlockchainStorage<B> + 'static,
 		GS: BuildStorage,
-		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+		E: CodeExecutor + RuntimeInfo,
 {
 	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, None);
 	let executor = GenesisCallExecutor::new(backend.clone(), local_executor);
@@ -76,7 +76,7 @@ pub fn new_fetch_checker<E, B: BlockT, S: BlockchainStorage<B>>(
 	executor: E,
 ) -> LightDataChecker<E, Blake2Hasher, B, S>
 	where
-		E: CodeExecutor<Blake2Hasher>,
+		E: CodeExecutor,
 {
 	LightDataChecker::new(blockchain, executor)
 }