Make runtime api calls native when possible (#1302)

* Add simple benchmark for the runtime api

* Make the executor support native calls

* Some documentation

* Hide behind `feature = "std"`

* Rework the native calls

* Make all tests compile again

* Make every parameter using the Block serialized/deserialized in the native call

* Forward `UnwindSafe` requirement

* Remove debug stuff

* Add some documentation

* Fixes warnings

* Fixes errors after master rebase

* Fixes compilation after master rebase

* Fixes compilation after rebase

substrate/core/client/src/block_builder/block_builder.rs

@@ -84,12 +84,12 @@ where
 	/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
 	pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> error::Result<()> {
 		use crate::runtime_api::ApiExt;
-		
+
 		let block_id = &self.block_id;
 		let extrinsics = &mut self.extrinsics;
-		
+
 		self.api.map_api_result(|api| {
-			match api.apply_extrinsic(block_id, &xt)? {
+			match api.apply_extrinsic(block_id, xt.clone())? {
 				Ok(ApplyOutcome::Success) | Ok(ApplyOutcome::Fail) => {
 					extrinsics.push(xt);
 					Ok(())

substrate/core/client/src/call_executor.rs

@@ -14,17 +14,17 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::sync::Arc;
-use std::cmp::Ord;
-use codec::Encode;
+use std::{sync::Arc, cmp::Ord, panic::UnwindSafe};
+use codec::{Encode, Decode};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::Block as BlockT;
-use state_machine::{self, OverlayedChanges, Ext,
-	CodeExecutor, ExecutionManager, native_when_possible};
+use state_machine::{
+	self, OverlayedChanges, Ext, CodeExecutor, ExecutionManager, native_when_possible
+};
 use executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use hash_db::Hasher;
 use trie::MemoryDB;
-use primitives::{H256, Blake2Hasher};
+use primitives::{H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue};
 
 use crate::backend;
 use crate::error;
@@ -56,7 +56,12 @@ where
 	/// of the execution context.
 	fn contextual_call<
 		PB: Fn() -> error::Result<B::Header>,
-		EM: Fn(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> R + UnwindSafe,
 	>(
 		&self,
 		at: &BlockId<B>,
@@ -66,7 +71,8 @@ where
 		initialised_block: &mut Option<BlockId<B>>,
 		prepare_environment_block: PB,
 		manager: ExecutionManager<EM>,
-	) -> error::Result<Vec<u8>> where ExecutionManager<EM>: Clone;
+		native_call: Option<NC>,
+	) -> error::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
 
 	/// Extract RuntimeVersion of given block
 	///
@@ -78,14 +84,20 @@ where
 	/// No changes are made.
 	fn call_at_state<
 		S: state_machine::Backend<H>,
-		F: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+		F: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> R + UnwindSafe,
 	>(&self,
 		state: &S,
 		overlay: &mut OverlayedChanges,
 		method: &str,
 		call_data: &[u8],
-		manager: ExecutionManager<F>
-	) -> Result<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>), error::Error>;
+		manager: ExecutionManager<F>,
+		native_call: Option<NC>,
+	) -> Result<(NativeOrEncoded<R>, S::Transaction, Option<MemoryDB<H>>), error::Error>;
 
 	/// Execute a call to a contract on top of given state, gathering execution proof.
 	///
@@ -155,7 +167,9 @@ where
 	) -> error::Result<Vec<u8>> {
 		let mut changes = OverlayedChanges::default();
 		let state = self.backend.state_at(*id)?;
-		let return_data = state_machine::execute_using_consensus_failure_handler(
+		let return_data = state_machine::execute_using_consensus_failure_handler::<
+			_, _, _, _, _, _, fn() -> NeverNativeValue
+		>(
 			&state,
 			self.backend.changes_trie_storage(),
 			&mut changes,
@@ -164,15 +178,21 @@ where
 			call_data,
 			native_when_possible(),
 			false,
+			None,
 		)
 		.map(|(result, _, _)| result)?;
 		self.backend.destroy_state(state)?;
-		Ok(return_data)
+		Ok(return_data.into_encoded())
 	}
 
 	fn contextual_call<
 		PB: Fn() -> error::Result<Block::Header>,
-		EM: Fn(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> R + UnwindSafe,
 	>(
 		&self,
 		at: &BlockId<Block>,
@@ -182,12 +202,15 @@ where
 		initialised_block: &mut Option<BlockId<Block>>,
 		prepare_environment_block: PB,
 		manager: ExecutionManager<EM>,
-	) -> Result<Vec<u8>, error::Error> where ExecutionManager<EM>: Clone {
+		native_call: Option<NC>,
+	) -> Result<NativeOrEncoded<R>, error::Error> where ExecutionManager<EM>: Clone {
 		let state = self.backend.state_at(*at)?;
 		//TODO: Find a better way to prevent double block initialization
 		if method != "Core_initialise_block" && initialised_block.map(|id| id != *at).unwrap_or(true) {
 			let header = prepare_environment_block()?;
-			state_machine::execute_using_consensus_failure_handler(
+			state_machine::execute_using_consensus_failure_handler::<
+				_, _, _, _, _, R, fn() -> R,
+			>(
 				&state,
 				self.backend.changes_trie_storage(),
 				changes,
@@ -196,6 +219,7 @@ where
 				&header.encode(),
 				manager.clone(),
 				false,
+				None,
 			)?;
 			*initialised_block = Some(*at);
 		}
@@ -209,8 +233,8 @@ where
 			call_data,
 			manager,
 			false,
-		)
-		.map(|(result, _, _)| result)?;
+			native_call,
+		).map(|(result, _, _)| result)?;
 
 		self.backend.destroy_state(state)?;
 		Ok(result)
@@ -226,14 +250,20 @@ where
 
 	fn call_at_state<
 		S: state_machine::Backend<Blake2Hasher>,
-		F: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+		F: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> R + UnwindSafe,
 	>(&self,
 		state: &S,
 		changes: &mut OverlayedChanges,
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
-	) -> error::Result<(Vec<u8>, S::Transaction, Option<MemoryDB<Blake2Hasher>>)> {
+		native_call: Option<NC>,
+	) -> error::Result<(NativeOrEncoded<R>, S::Transaction, Option<MemoryDB<Blake2Hasher>>)> {
 		state_machine::execute_using_consensus_failure_handler(
 			state,
 			self.backend.changes_trie_storage(),
@@ -243,6 +273,7 @@ where
 			call_data,
 			manager,
 			true,
+			native_call,
 		)
 		.map(|(result, storage_tx, changes_tx)| (
 			result,

substrate/core/client/src/client.rs

@@ -16,25 +16,29 @@
 
 //! Substrate Client
 
-use std::{marker::PhantomData, collections::{HashSet, BTreeMap}, sync::Arc};
+use std::{marker::PhantomData, collections::{HashSet, BTreeMap}, sync::Arc, panic::UnwindSafe};
 use crate::error::Error;
 use futures::sync::mpsc;
 use parking_lot::{Mutex, RwLock};
+use primitives::NativeOrEncoded;
 use runtime_primitives::{
 	Justification,
 	generic::{BlockId, SignedBlock},
 };
-use consensus::{Error as ConsensusError, ErrorKind as ConsensusErrorKind, ImportBlock, ImportResult, BlockOrigin, ForkChoiceStrategy};
+use consensus::{
+	Error as ConsensusError, ErrorKind as ConsensusErrorKind, ImportBlock, ImportResult,
+	BlockOrigin, ForkChoiceStrategy
+};
 use runtime_primitives::traits::{
 	Block as BlockT, Header as HeaderT, Zero, As, NumberFor, CurrentHeight, BlockNumberToHash,
 	ApiRef, ProvideRuntimeApi, Digest, DigestItem, AuthorityIdFor
 };
 use runtime_primitives::BuildStorage;
-use crate::runtime_api::{Core as CoreAPI, CallRuntimeAt, ConstructRuntimeApi};
-use primitives::{Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash};
+use crate::runtime_api::{CallRuntimeAt, ConstructRuntimeApi};
+use primitives::{Blake2Hasher, H256, ChangesTrieConfiguration, convert_hash, NeverNativeValue};
 use primitives::storage::{StorageKey, StorageData};
 use primitives::storage::well_known_keys;
-use codec::Decode;
+use codec::{Encode, Decode};
 use state_machine::{
 	DBValue, Backend as StateBackend, CodeExecutor, ChangesTrieAnchorBlockId,
 	ExecutionStrategy, ExecutionManager, prove_read,
@@ -557,7 +561,9 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		&self
 	) -> error::Result<block_builder::BlockBuilder<Block, InherentData, Self>> where
 		E: Clone + Send + Sync,
-		RA: BlockBuilderAPI<Block, InherentData>
+		RA: Send + Sync,
+		Self: ProvideRuntimeApi,
+		<Self as ProvideRuntimeApi>::Api: BlockBuilderAPI<Block, InherentData>
 	{
 		block_builder::BlockBuilder::new(self)
 	}
@@ -567,7 +573,9 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		&self, parent: &BlockId<Block>
 	) -> error::Result<block_builder::BlockBuilder<Block, InherentData, Self>> where
 		E: Clone + Send + Sync,
-		RA: BlockBuilderAPI<Block, InherentData>
+		RA: Send + Sync,
+		Self: ProvideRuntimeApi,
+		<Self as ProvideRuntimeApi>::Api: BlockBuilderAPI<Block, InherentData>
 	{
 		block_builder::BlockBuilder::at_block(parent, &self)
 	}
@@ -615,7 +623,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		let (storage_update, changes_update, storage_changes) = match transaction.state()? {
 			Some(transaction_state) => {
 				let mut overlay = Default::default();
-				let r = self.executor.call_at_state(
+				let r = self.executor.call_at_state::<_, _, NeverNativeValue, fn() -> NeverNativeValue>(
 					transaction_state,
 					&mut overlay,
 					"Core_execute_block",
@@ -638,6 +646,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 							wasm_result
 						}),
 					},
+					None,
 				);
 				let (_, storage_update, changes_update) = r?;
 				overlay.commit_prospective();
@@ -1017,30 +1026,29 @@ impl<B, E, Block, RA> ProvideRuntimeApi for Client<B, E, Block, RA> where
 	B: backend::Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher> + Clone + Send + Sync,
 	Block: BlockT<Hash=H256>,
-	RA: CoreAPI<Block>
+	RA: ConstructRuntimeApi<Block, Self>
 {
-	type Api = RA;
+	type Api = <RA as ConstructRuntimeApi<Block, Self>>::RuntimeApi;
 
 	fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> {
-		Self::Api::construct_runtime_api(self)
+		RA::construct_runtime_api(self)
 	}
 }
 
 impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 	B: backend::Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher> + Clone + Send + Sync,
-	Block: BlockT<Hash=H256>,
-	RA: CoreAPI<Block>, // not strictly necessary at the moment
-						// but we want to bound to make sure the API is actually available.
+	Block: BlockT<Hash=H256>
 {
-	fn call_api_at(
+	fn call_api_at<R: Encode + Decode + PartialEq, NC: FnOnce() -> R + UnwindSafe>(
 		&self,
 		at: &BlockId<Block>,
 		function: &'static str,
 		args: Vec<u8>,
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
-	) -> error::Result<Vec<u8>> {
+		native_call: Option<NC>,
+	) -> error::Result<NativeOrEncoded<R>> {
 		let execution_manager = match self.api_execution_strategy {
 			ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
 			ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm,
@@ -1053,8 +1061,16 @@ impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 			}),
 		};
 
-		self.executor.contextual_call(at, function, &args, changes, initialised_block,
-			|| self.prepare_environment_block(at), execution_manager)
+		self.executor.contextual_call(
+			at,
+			function,
+			&args,
+			changes,
+			initialised_block,
+			|| self.prepare_environment_block(at),
+			execution_manager,
+			native_call,
+		)
 	}
 
 	fn runtime_version_at(&self, at: &BlockId<Block>) -> error::Result<RuntimeVersion> {
@@ -1268,7 +1284,7 @@ pub(crate) mod tests {
 	use consensus::BlockOrigin;
 	use test_client::client::{backend::Backend as TestBackend, runtime_api::ApiExt};
 	use test_client::BlockBuilderExt;
-	use test_client::runtime::{self, Block, Transfer, RuntimeApi, test_api::TestAPI};
+	use test_client::runtime::{self, Block, Transfer, RuntimeApi, TestAPI};
 
 	/// Returns tuple, consisting of:
 	/// 1) test client pre-filled with blocks changing balances;
@@ -1350,14 +1366,14 @@ pub(crate) mod tests {
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				&Keyring::Alice.to_raw_public().into()
+				Keyring::Alice.to_raw_public().into()
 			).unwrap(),
 			1000
 		);
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				&Keyring::Ferdie.to_raw_public().into()
+				Keyring::Ferdie.to_raw_public().into()
 			).unwrap(),
 			0
 		);
@@ -1417,14 +1433,14 @@ pub(crate) mod tests {
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				&Keyring::Alice.to_raw_public().into()
+				Keyring::Alice.to_raw_public().into()
 			).unwrap(),
 			958
 		);
 		assert_eq!(
 			client.runtime_api().balance_of(
 				&BlockId::Number(client.info().unwrap().chain.best_number),
-				&Keyring::Ferdie.to_raw_public().into()
+				Keyring::Ferdie.to_raw_public().into()
 			).unwrap(),
 			42
 		);

substrate/core/client/src/light/call_executor.rs

@@ -17,13 +17,11 @@
 //! Light client call exector. Executes methods on remote full nodes, fetching
 //! execution proof and checking it locally.
 
-use std::collections::HashSet;
-use std::marker::PhantomData;
-use std::sync::Arc;
+use std::{collections::HashSet, marker::PhantomData, sync::Arc};
 use futures::{IntoFuture, Future};
 
 use codec::{Encode, Decode};
-use primitives::{H256, Blake2Hasher, convert_hash};
+use primitives::{H256, Blake2Hasher, convert_hash, NativeOrEncoded};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{As, Block as BlockT, Header as HeaderT};
 use state_machine::{self, Backend as StateBackend, CodeExecutor, OverlayedChanges,
@@ -88,7 +86,12 @@ where
 
 	fn contextual_call<
 		PB: Fn() -> ClientResult<Block::Header>,
-		EM: Fn(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC,
 	>(
 		&self,
 		at: &BlockId<Block>,
@@ -98,13 +101,14 @@ where
 		initialised_block: &mut Option<BlockId<Block>>,
 		_prepare_environment_block: PB,
 		_manager: ExecutionManager<EM>,
-	) -> ClientResult<Vec<u8>> where ExecutionManager<EM>: Clone {
+		_native_call: Option<NC>,
+	) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
 		// it is only possible to execute contextual call if changes are empty
 		if !changes.is_empty() || initialised_block.is_some() {
 			return Err(ClientErrorKind::NotAvailableOnLightClient.into());
 		}
 
-		self.call(at, method, call_data)
+		self.call(at, method, call_data).map(NativeOrEncoded::Encoded)
 	}
 
 	fn runtime_version(&self, id: &BlockId<Block>) -> ClientResult<RuntimeVersion> {
@@ -115,14 +119,20 @@ where
 
 	fn call_at_state<
 		S: StateBackend<H>,
-		FF: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>
+		FF: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> R,
 	>(&self,
 		_state: &S,
 		_changes: &mut OverlayedChanges,
 		_method: &str,
 		_call_data: &[u8],
-		_m: ExecutionManager<FF>
-	) -> ClientResult<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>)> {
+		_m: ExecutionManager<FF>,
+		_native_call: Option<NC>,
+	) -> ClientResult<(NativeOrEncoded<R>, S::Transaction, Option<MemoryDB<H>>)> {
 		Err(ClientErrorKind::NotAvailableOnLightClient.into())
 	}
 

substrate/core/client/src/runtime_api.rs

@@ -20,6 +20,9 @@
 #[cfg(feature = "std")]
 pub use state_machine::OverlayedChanges;
 #[doc(hidden)]
+#[cfg(feature = "std")]
+pub use primitives::NativeOrEncoded;
+#[doc(hidden)]
 pub use runtime_primitives::{
 	traits::{AuthorityIdFor, Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, ApiRef, RuntimeApiInfo},
 	generic::BlockId, transaction_validity::TransactionValidity
@@ -34,16 +37,19 @@ pub use codec::{Encode, Decode};
 #[cfg(feature = "std")]
 use crate::error;
 use rstd::vec::Vec;
-use primitives::OpaqueMetadata;
 use sr_api_macros::decl_runtime_apis;
+use primitives::OpaqueMetadata;
+#[cfg(feature = "std")]
+use std::panic::UnwindSafe;
 
 /// Something that can be constructed to a runtime api.
 #[cfg(feature = "std")]
-pub trait ConstructRuntimeApi<Block: BlockT> {
+pub trait ConstructRuntimeApi<Block: BlockT, C: CallRuntimeAt<Block>> {
+	/// The actual runtime api that will be constructed.
+	type RuntimeApi;
+
 	/// Construct an instance of the runtime api.
-	fn construct_runtime_api<'a, T: CallRuntimeAt<Block>>(
-		call: &'a T
-	) -> ApiRef<'a, Self> where Self: Sized;
+	fn construct_runtime_api<'a>(call: &'a C) -> ApiRef<'a, Self::RuntimeApi>;
 }
 
 /// An extension for the `RuntimeApi`.
@@ -71,14 +77,15 @@ pub trait ApiExt<Block: BlockT> {
 pub trait CallRuntimeAt<Block: BlockT> {
 	/// Calls the given api function with the given encoded arguments at the given block
 	/// and returns the encoded result.
-	fn call_api_at(
+	fn call_api_at<R: Encode + Decode + PartialEq, NC: FnOnce() -> R + UnwindSafe>(
 		&self,
 		at: &BlockId<Block>,
 		function: &'static str,
 		args: Vec<u8>,
 		changes: &mut OverlayedChanges,
 		initialised_block: &mut Option<BlockId<Block>>,
-	) -> error::Result<Vec<u8>>;
+		native_call: Option<NC>,
+	) -> error::Result<NativeOrEncoded<R>>;
 
 	/// Returns the runtime version at the given block.
 	fn runtime_version_at(&self, at: &BlockId<Block>) -> error::Result<RuntimeVersion>;
@@ -95,7 +102,7 @@ decl_runtime_apis! {
 		/// Execute the given block.
 		fn execute_block(block: Block);
 		/// Initialise a block with the given header.
-		fn initialise_block(header: <Block as BlockT>::Header);
+		fn initialise_block(header: &<Block as BlockT>::Header);
 	}
 
 	/// The `Metadata` api trait that returns metadata for the runtime.