Adds new execution strategy nativeElseWasm (#1546)

* fix: adds new execution strategy nativeElseWasm and replace nativeWhenPossible with it

* feat: adds cmd line params for execution strategies

* fix: uses of cmd line execution strategies

* chore: remove white spaces

* chore: remove println

* chore: remove whitespace

* fix: generating functions with context

* feat: add function to generate with_context declarations

* fix: add implementation for with_context function calls

* fix: add execution context to call_api_at function

* fix: making use of context to select strategy for block_builder

* chore: cleaning up

* fix: merging issues

* fix tests

* add wasm files

* chore: small doc for context fields

* chore: delete redundant docs

* fix: use full path for ExecutionContext

* fix: add context functions from inside fold_item_impl

* chore: remove clone

* fix: moving generative function to utils, remove unused imports

* fix: add missing full path for ExecutionContext

* fix: merge issues

* update wasm files

* fix: update to keep up with changes in master

* chore: remove unused functions, clean up

* fix test

* fix grumbles

* fix: add more tests

* fix: some refactorings

* feat: add execution strategy to call

* chore: small improvements

* fix: add message to panic

* fix tests

substrate/core/client/src/block_builder/block_builder.rs

@@ -25,7 +25,7 @@ use primitives::H256;
 use runtime_primitives::generic::BlockId;
 use crate::runtime_api::Core;
 use crate::error;
-use runtime_primitives::ApplyOutcome;
+use runtime_primitives::{ApplyOutcome, ExecutionContext};
 
 
 /// Utility for building new (valid) blocks from a stream of extrinsics.
@@ -56,7 +56,6 @@ where
 
 		let parent_hash = api.block_hash_from_id(block_id)?
 			.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", block_id)))?;
-
 		let header = <<Block as BlockT>::Header as HeaderT>::new(
 			number,
 			Default::default(),
@@ -64,10 +63,8 @@ where
 			parent_hash,
 			Default::default()
 		);
-
 		let api = api.runtime_api();
-		api.initialise_block(block_id, &header)?;
-
+		api.initialise_block_with_context(block_id, ExecutionContext::BlockConstruction, &header)?;
 		Ok(BlockBuilder {
 			header,
 			extrinsics: Vec::new(),
@@ -86,7 +83,7 @@ where
 		let extrinsics = &mut self.extrinsics;
 
 		self.api.map_api_result(|api| {
-			match api.apply_extrinsic(block_id, xt.clone())? {
+			match api.apply_extrinsic_with_context(block_id, ExecutionContext::BlockConstruction, xt.clone())? {
 				Ok(ApplyOutcome::Success) | Ok(ApplyOutcome::Fail) => {
 					extrinsics.push(xt);
 					Ok(())
@@ -100,7 +97,7 @@ where
 
 	/// Consume the builder to return a valid `Block` containing all pushed extrinsics.
 	pub fn bake(mut self) -> error::Result<Block> {
-		self.header = self.api.finalise_block(&self.block_id)?;
+		self.header = self.api.finalise_block_with_context(&self.block_id, ExecutionContext::BlockConstruction)?;
 
 		debug_assert_eq!(
 			self.header.extrinsics_root().clone(),

substrate/core/client/src/call_executor.rs

@@ -19,7 +19,7 @@ use codec::{Encode, Decode};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::Block as BlockT;
 use state_machine::{
-	self, OverlayedChanges, Ext, CodeExecutor, ExecutionManager, native_when_possible
+	self, OverlayedChanges, Ext, CodeExecutor, ExecutionManager, ExecutionStrategy
 };
 use executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use hash_db::Hasher;
@@ -47,6 +47,7 @@ where
 		id: &BlockId<B>,
 		method: &str,
 		call_data: &[u8],
+		strategy: ExecutionStrategy,
 	) -> Result<Vec<u8>, error::Error>;
 
 	/// Execute a contextual call on top of state in a block of a given hash.
@@ -70,7 +71,7 @@ where
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<B>>,
 		prepare_environment_block: PB,
-		manager: ExecutionManager<EM>,
+		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
 	) -> error::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
 
@@ -164,6 +165,7 @@ where
 		id: &BlockId<Block>,
 		method: &str,
 		call_data: &[u8],
+		strategy: ExecutionStrategy
 	) -> error::Result<Vec<u8>> {
 		let mut changes = OverlayedChanges::default();
 		let state = self.backend.state_at(*id)?;
@@ -176,7 +178,7 @@ where
 			&self.executor,
 			method,
 			call_data,
-			native_when_possible(),
+			strategy.get_manager(),
 			false,
 			None,
 		)
@@ -201,7 +203,7 @@ where
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
 		prepare_environment_block: PB,
-		manager: ExecutionManager<EM>,
+		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
 	) -> Result<NativeOrEncoded<R>, error::Error> where ExecutionManager<EM>: Clone {
 		let state = self.backend.state_at(*at)?;
@@ -216,7 +218,7 @@ where
 				&self.executor,
 				"Core_initialise_block",
 				&header.encode(),
-				manager.clone(),
+				execution_manager.clone(),
 				false,
 				None,
 			)?;
@@ -230,7 +232,7 @@ where
 			&self.executor,
 			method,
 			call_data,
-			manager,
+			execution_manager,
 			false,
 			native_call,
 		).map(|(result, _, _)| result)?;

substrate/core/client/src/client.rs

@@ -33,7 +33,7 @@ use runtime_primitives::traits::{
 	Block as BlockT, Header as HeaderT, Zero, As, NumberFor, CurrentHeight, BlockNumberToHash,
 	ApiRef, ProvideRuntimeApi, Digest, DigestItem, AuthorityIdFor
 };
-use runtime_primitives::BuildStorage;
+use runtime_primitives::{BuildStorage, ExecutionContext};
 use crate::runtime_api::{CallRuntimeAt, ConstructRuntimeApi};
 use primitives::{Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, NeverNativeValue};
 use primitives::storage::{StorageKey, StorageData};
@@ -75,6 +75,30 @@ pub type FinalityNotifications<Block> = mpsc::UnboundedReceiver<FinalityNotifica
 type StorageUpdate<B, Block> = <<<B as backend::Backend<Block, Blake2Hasher>>::BlockImportOperation as BlockImportOperation<Block, Blake2Hasher>>::State as state_machine::Backend<Blake2Hasher>>::Transaction;
 type ChangesUpdate = trie::MemoryDB<Blake2Hasher>;
 
+/// Execution strategies settings.
+#[derive(Debug, Clone)]
+pub struct ExecutionStrategies {
+	/// Execution strategy used when syncing.
+	pub syncing: ExecutionStrategy,
+	/// Execution strategy used when importing blocks.
+	pub importing: ExecutionStrategy,
+	/// Execution strategy used when constructing blocks.
+	pub block_construction: ExecutionStrategy,
+	/// Execution strategy used in other cases.
+	pub other: ExecutionStrategy,
+}
+
+impl Default for ExecutionStrategies {
+	fn default() -> ExecutionStrategies {
+		ExecutionStrategies {
+			syncing: ExecutionStrategy::NativeElseWasm,
+			importing: ExecutionStrategy::NativeElseWasm,
+			block_construction: ExecutionStrategy::AlwaysWasm,
+			other: ExecutionStrategy::NativeElseWasm,
+		}
+	}
+}
+
 /// Substrate Client
 pub struct Client<B, E, Block, RA> where Block: BlockT {
 	backend: Arc<B>,
@@ -85,8 +109,7 @@ pub struct Client<B, E, Block, RA> where Block: BlockT {
 	import_lock: Mutex<()>,
 	// holds the block hash currently being imported. TODO: replace this with block queue
 	importing_block: RwLock<Option<Block::Hash>>,
-	block_execution_strategy: ExecutionStrategy,
-	api_execution_strategy: ExecutionStrategy,
+	execution_strategies: ExecutionStrategies,
 	_phantom: PhantomData<RA>,
 }
 
@@ -236,7 +259,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 		B: backend::LocalBackend<Block, Blake2Hasher>
 {
 	let call_executor = LocalCallExecutor::new(backend.clone(), executor);
-	Client::new(backend, call_executor, build_genesis_storage, ExecutionStrategy::NativeWhenPossible, ExecutionStrategy::NativeWhenPossible)
+	Client::new(backend, call_executor, build_genesis_storage, Default::default())
 }
 
 impl<B, E, Block, RA> Client<B, E, Block, RA> where
@@ -249,8 +272,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		backend: Arc<B>,
 		executor: E,
 		build_genesis_storage: S,
-		block_execution_strategy: ExecutionStrategy,
-		api_execution_strategy: ExecutionStrategy,
+		execution_strategies: ExecutionStrategies
 	) -> error::Result<Self> {
 		if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() {
 			let (genesis_storage, children_genesis_storage) = build_genesis_storage.build_storage()?;
@@ -276,12 +298,16 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			finality_notification_sinks: Default::default(),
 			import_lock: Default::default(),
 			importing_block: Default::default(),
-			block_execution_strategy,
-			api_execution_strategy,
+			execution_strategies,
 			_phantom: Default::default(),
 		})
 	}
 
+	/// Get a reference to the execution strategies.
+	pub fn execution_strategies(&self) -> &ExecutionStrategies {
+		&self.execution_strategies
+	}
+
 	/// Get a reference to the state at a given block.
 	pub fn state_at(&self, block: &BlockId<Block>) -> error::Result<B::State> {
 		self.backend.state_at(*block)
@@ -315,7 +341,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 	pub fn authorities_at(&self, id: &BlockId<Block>) -> error::Result<Vec<AuthorityIdFor<Block>>> {
 		match self.backend.blockchain().cache().and_then(|cache| cache.authorities_at(*id)) {
 			Some(cached_value) => Ok(cached_value),
-			None => self.executor.call(id, "Core_authorities", &[])
+			None => self.executor.call(id, "Core_authorities", &[], ExecutionStrategy::NativeElseWasm)
 				.and_then(|r| Vec::<AuthorityIdFor<Block>>::decode(&mut &r[..])
 					.ok_or_else(|| error::ErrorKind::InvalidAuthoritiesSet.into()))
 		}
@@ -803,16 +829,12 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		match transaction.state()? {
 			Some(transaction_state) => {
 				let mut overlay = Default::default();
-				let (_, storage_update, changes_update) = self.executor.call_at_state::<_, _, NeverNativeValue, fn() -> _>(
-					transaction_state,
-					&mut overlay,
-					"Core_execute_block",
-					&<Block as BlockT>::new(import_headers.pre().clone(), body.unwrap_or_default()).encode(),
-					match (origin, self.block_execution_strategy) {
-						(BlockOrigin::NetworkInitialSync, _) | (_, ExecutionStrategy::NativeWhenPossible) =>
-							ExecutionManager::NativeWhenPossible,
-						(_, ExecutionStrategy::AlwaysWasm) => ExecutionManager::AlwaysWasm,
-						_ => ExecutionManager::Both(|wasm_result, native_result| {
+				let get_execution_manager = |execution_strategy: ExecutionStrategy| {
+					match execution_strategy {
+						ExecutionStrategy::NativeElseWasm => ExecutionManager::NativeElseWasm,
+						ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm,
+						ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
+						ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| {
 							let header = import_headers.post();
 							warn!("Consensus error between wasm and native block execution at block {}", hash);
 							warn!("   Header {:?}", header);
@@ -825,6 +847,16 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 							);
 							wasm_result
 						}),
+					}
+				};
+				let (_, storage_update, changes_update) = self.executor.call_at_state::<_, _, NeverNativeValue, fn() -> _>(
+					transaction_state,
+					&mut overlay,
+					"Core_execute_block",
+					&<Block as BlockT>::new(import_headers.pre().clone(), body.unwrap_or_default()).encode(),
+					match origin {
+						BlockOrigin::NetworkInitialSync => get_execution_manager(self.execution_strategies().syncing),
+						_ => get_execution_manager(self.execution_strategies().importing),
 					},
 					None,
 				)?;
@@ -1253,27 +1285,22 @@ impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
 		native_call: Option<NC>,
+		context: ExecutionContext
 	) -> error::Result<NativeOrEncoded<R>> {
-		let execution_manager = match self.api_execution_strategy {
-			ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
-			ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm,
-			ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| {
-				warn!("Consensus error between wasm and native runtime execution at block {:?}", at);
-				warn!("   Function {:?}", function);
-				warn!("   Native result {:?}", native_result);
-				warn!("   Wasm result {:?}", wasm_result);
-				wasm_result
-			}),
+		let manager = match context {
+			ExecutionContext::BlockConstruction => self.execution_strategies.block_construction.get_manager(),
+			ExecutionContext::Syncing => self.execution_strategies.syncing.get_manager(),
+			ExecutionContext::Importing => self.execution_strategies.importing.get_manager(),
+			ExecutionContext::Other => self.execution_strategies.other.get_manager(),
 		};
-
-		self.executor.contextual_call(
+		self.executor.contextual_call::<_, fn(_,_) -> _,_,_>(
 			at,
 			function,
 			&args,
 			changes,
 			initialised_block,
 			|| self.prepare_environment_block(at),
-			execution_manager,
+			manager,
 			native_call,
 		)
 	}

substrate/core/client/src/genesis.rs

@@ -93,7 +93,7 @@ mod tests {
 			&executor(),
 			"Core_initialise_block",
 			&header.encode(),
-			ExecutionStrategy::NativeWhenPossible,
+			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
 
 		for tx in transactions.iter() {
@@ -104,7 +104,7 @@ mod tests {
 				&executor(),
 				"BlockBuilder_apply_extrinsic",
 				&tx.encode(),
-				ExecutionStrategy::NativeWhenPossible,
+				ExecutionStrategy::NativeElseWasm,
 			).unwrap();
 		}
 
@@ -115,7 +115,7 @@ mod tests {
 			&executor(),
 			"BlockBuilder_finalise_block",
 			&[],
-			ExecutionStrategy::NativeWhenPossible,
+			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
 		header = Header::decode(&mut &ret_data[..]).unwrap();
 		println!("root after: {:?}", header.extrinsics_root);
@@ -159,7 +159,7 @@ mod tests {
 			&executor(),
 			"Core_execute_block",
 			&b1data,
-			ExecutionStrategy::NativeWhenPossible,
+			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
 	}
 
@@ -210,7 +210,7 @@ mod tests {
 			&Executor::new(None),
 			"Core_execute_block",
 			&b1data,
-			ExecutionStrategy::NativeWhenPossible,
+			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
 	}
 }

substrate/core/client/src/lib.rs

@@ -57,7 +57,7 @@ pub use crate::client::{
 	new_with_backend,
 	new_in_mem,
 	BlockBody, BlockStatus, ImportNotifications, FinalityNotifications, BlockchainEvents,
-	BlockImportNotification, Client, ClientInfo, ChainHead,
+	BlockImportNotification, Client, ClientInfo, ChainHead, ExecutionStrategies,
 };
 #[cfg(feature = "std")]
 pub use crate::notifications::{StorageEventStream, StorageChangeSet};

substrate/core/client/src/light/call_executor.rs

@@ -24,7 +24,7 @@ use codec::{Encode, Decode};
 use primitives::{H256, Blake2Hasher, convert_hash, NativeOrEncoded};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{As, Block as BlockT, Header as HeaderT};
-use state_machine::{self, Backend as StateBackend, CodeExecutor, OverlayedChanges,
+use state_machine::{self, Backend as StateBackend, CodeExecutor, OverlayedChanges, ExecutionStrategy,
 	create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionManager};
 use hash_db::Hasher;
 
@@ -80,7 +80,8 @@ where
 {
 	type Error = ClientError;
 
-	fn call(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> ClientResult<Vec<u8>> {
+	fn call(&self, id: &BlockId<Block>, method: &str, call_data: &[u8], _strategy: ExecutionStrategy)
+		-> ClientResult<Vec<u8>> {
 		let block_hash = self.blockchain.expect_block_hash_from_id(id)?;
 		let block_header = self.blockchain.expect_header(id.clone())?;
 
@@ -109,7 +110,7 @@ where
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
 		_prepare_environment_block: PB,
-		_manager: ExecutionManager<EM>,
+		execution_manager: ExecutionManager<EM>,
 		_native_call: Option<NC>,
 	) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
 		// it is only possible to execute contextual call if changes are empty
@@ -117,11 +118,11 @@ where
 			return Err(ClientErrorKind::NotAvailableOnLightClient.into());
 		}
 
-		self.call(at, method, call_data).map(NativeOrEncoded::Encoded)
+		self.call(at, method, call_data, (&execution_manager).into()).map(NativeOrEncoded::Encoded)
 	}
 
 	fn runtime_version(&self, id: &BlockId<Block>) -> ClientResult<RuntimeVersion> {
-		let call_result = self.call(id, "version", &[])?;
+		let call_result = self.call(id, "version", &[], ExecutionStrategy::NativeElseWasm)?;
 		RuntimeVersion::decode(&mut call_result.as_slice())
 			.ok_or_else(|| ClientErrorKind::VersionInvalid.into())
 	}
@@ -200,10 +201,11 @@ impl<Block, B, Remote, Local> CallExecutor<Block, Blake2Hasher> for
 {
 	type Error = ClientError;
 
-	fn call(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> ClientResult<Vec<u8>> {
+	fn call(&self, id: &BlockId<Block>, method: &str, call_data: &[u8], strategy: ExecutionStrategy)
+		-> ClientResult<Vec<u8>> {
 		match self.backend.is_local_state_available(id) {
-			true => self.local.call(id, method, call_data),
-			false => self.remote.call(id, method, call_data),
+			true => self.local.call(id, method, call_data, strategy),
+			false => self.remote.call(id, method, call_data, strategy),
 		}
 	}
 
@@ -507,7 +509,7 @@ mod tests {
 		let local_executor = RemoteCallExecutor::new(Arc::new(backend.blockchain().clone()), Arc::new(OkCallFetcher::new(vec![1])));
 		let remote_executor = RemoteCallExecutor::new(Arc::new(backend.blockchain().clone()), Arc::new(OkCallFetcher::new(vec![2])));
 		let remote_or_local = RemoteOrLocalCallExecutor::new(backend, remote_executor, local_executor);
-		assert_eq!(remote_or_local.call(&BlockId::Number(0), "test_method", &[]).unwrap(), vec![1]);
-		assert_eq!(remote_or_local.call(&BlockId::Number(1), "test_method", &[]).unwrap(), vec![2]);
+		assert_eq!(remote_or_local.call(&BlockId::Number(0), "test_method", &[], ExecutionStrategy::NativeElseWasm).unwrap(), vec![1]);
+		assert_eq!(remote_or_local.call(&BlockId::Number(1), "test_method", &[], ExecutionStrategy::NativeElseWasm).unwrap(), vec![2]);
 	}
 }

substrate/core/client/src/light/mod.rs

@@ -27,7 +27,7 @@ use executor::RuntimeInfo;
 use primitives::{H256, Blake2Hasher};
 use runtime_primitives::BuildStorage;
 use runtime_primitives::traits::Block as BlockT;
-use state_machine::{CodeExecutor, ExecutionStrategy};
+use state_machine::CodeExecutor;
 
 use crate::call_executor::LocalCallExecutor;
 use crate::client::Client;
@@ -75,7 +75,7 @@ pub fn new_light<B, S, F, GS, RA, E>(
 	let remote_executor = RemoteCallExecutor::new(backend.blockchain().clone(), fetcher);
 	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor);
 	let executor = RemoteOrLocalCallExecutor::new(backend.clone(), remote_executor, local_executor);
-	Client::new(backend, executor, genesis_storage, ExecutionStrategy::NativeWhenPossible, ExecutionStrategy::NativeWhenPossible)
+	Client::new(backend, executor, genesis_storage, Default::default())
 }
 
 /// Create an instance of fetch data checker.

substrate/core/client/src/runtime_api.rs

@@ -25,7 +25,7 @@ pub use primitives::NativeOrEncoded;
 #[doc(hidden)]
 pub use runtime_primitives::{
 	traits::{AuthorityIdFor, Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, ApiRef, RuntimeApiInfo},
-	generic::BlockId, transaction_validity::TransactionValidity
+	generic::BlockId, transaction_validity::TransactionValidity, ExecutionContext,
 };
 #[doc(hidden)]
 pub use runtime_version::{ApiId, RuntimeVersion, ApisVec, create_apis_vec};
@@ -99,6 +99,7 @@ pub trait CallRuntimeAt<Block: BlockT> {
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
 		native_call: Option<NC>,
+		context: ExecutionContext
 	) -> error::Result<NativeOrEncoded<R>>;
 
 	/// Returns the runtime version at the given block.