Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/primitives/api/Cargo.toml

@@ -12,6 +12,7 @@ sp-std = { version = "2.0.0", default-features = false, path = "../std" }
 sp-runtime = { version = "2.0.0", default-features = false, path = "../runtime" }
 sp-version = { version = "2.0.0", default-features = false, path = "../version" }
 sp-state-machine = { version = "2.0.0", optional = true, path = "../../primitives/state-machine" }
+hash-db = { version = "0.15.2", optional = true }
 
 [dev-dependencies]
 sp-test-primitives = { version = "2.0.0", path = "../test-primitives" }
@@ -25,4 +26,5 @@ std = [
 	"sp-runtime/std",
 	"sp-state-machine",
 	"sp-version/std",
+	"hash-db",
 ]

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -16,7 +16,7 @@
 
 use crate::utils::{
 	generate_crate_access, generate_hidden_includes, generate_runtime_mod_name_for_trait,
-	fold_fn_decl_for_client_side, unwrap_or_error, extract_parameter_names_types_and_borrows,
+	fold_fn_decl_for_client_side, extract_parameter_names_types_and_borrows,
 	generate_native_call_generator_fn_name, return_type_extract_type,
 	generate_method_runtime_api_impl_name, generate_call_api_at_fn_name, prefix_function_with_trait,
 	replace_wild_card_parameter_names,
@@ -399,7 +399,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				R: #crate_::Encode + #crate_::Decode + PartialEq,
 				NC: FnOnce() -> std::result::Result<R, String> + std::panic::UnwindSafe,
 				Block: #crate_::BlockT,
-				T: #crate_::CallRuntimeAt<Block>,
+				T: #crate_::CallApiAt<Block>,
 				C: #crate_::Core<Block, Error = T::Error>,
 			>(
 				call_runtime_at: &T,
@@ -407,6 +407,9 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
+				storage_transaction_cache: &std::cell::RefCell<
+					#crate_::StorageTransactionCache<Block, T::StateBackend>
+				>,
 				initialized_block: &std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 				native_call: Option<NC>,
 				context: #crate_::ExecutionContext,
@@ -426,34 +429,40 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 					if version.apis.iter().any(|(s, v)| {
 						s == &ID && *v < #versions
 					}) {
-						let ret = call_runtime_at.call_api_at::<R, fn() -> _, _>(
+						let params = #crate_::CallApiAtParams::<_, _, fn() -> _, _> {
 							core_api,
 							at,
-							#old_names,
-							args,
-							changes,
+							function: #old_names,
+							native_call: None,
+							arguments: args,
+							overlayed_changes: changes,
+							storage_transaction_cache,
 							initialize_block,
-							None,
 							context,
 							recorder,
-						)?;
+						};
+
+						let ret = call_runtime_at.call_api_at(params)?;
 
 						update_initialized_block();
-						return Ok(ret);
+						return Ok(ret)
 					}
 				)*
 
-				let ret = call_runtime_at.call_api_at(
+				let params = #crate_::CallApiAtParams {
 					core_api,
 					at,
-					#trait_fn_name,
-					args,
-					changes,
-					initialize_block,
+					function: #trait_fn_name,
 					native_call,
+					arguments: args,
+					overlayed_changes: changes,
+					storage_transaction_cache,
+					initialize_block,
 					context,
 					recorder,
-				)?;
+				};
+
+				let ret = call_runtime_at.call_api_at(params)?;
 
 				update_initialized_block();
 				Ok(ret)
@@ -465,7 +474,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 }
 
 /// Generate the declaration of the trait for the runtime.
-fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
+fn generate_runtime_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 	let mut result = Vec::new();
 
 	for decl in decls {
@@ -473,12 +482,12 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 		extend_generics_with_block(&mut decl.generics);
 		let mod_name = generate_runtime_mod_name_for_trait(&decl.ident);
 		let found_attributes = remove_supported_attributes(&mut decl.attrs);
-		let api_version = unwrap_or_error(get_api_version(&found_attributes).map(|v| {
+		let api_version = get_api_version(&found_attributes).map(|v| {
 			generate_runtime_api_version(v as u32)
-		}));
+		})?;
 		let id = generate_runtime_api_id(&decl.ident.to_string());
 
-		let call_api_at_calls = unwrap_or_error(generate_call_api_at_calls(&decl));
+		let call_api_at_calls = generate_call_api_at_calls(&decl)?;
 
 		// Remove methods that have the `changed_in` attribute as they are not required for the
 		// runtime anymore.
@@ -495,7 +504,7 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 			r => Some(r.clone()),
 		}).collect();
 
-		let native_call_generators = unwrap_or_error(generate_native_call_generators(&decl));
+		let native_call_generators = generate_native_call_generators(&decl)?;
 
 		result.push(quote!(
 			#[doc(hidden)]
@@ -517,7 +526,7 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 		));
 	}
 
-	quote!( #( #result )* )
+	Ok(quote!( #( #result )* ))
 }
 
 /// Modify the given runtime api declaration to be usable on the client side.
@@ -722,7 +731,7 @@ impl<'a> Fold for ToClientSideDecl<'a> {
 				'static
 				+ Send
 				+ Sync
-				+ #crate_::ApiExt<#block_ident>
+				+ #crate_::ApiErrorExt
 			);
 		} else {
 			// Add the `Core` runtime api as super trait.
@@ -823,7 +832,7 @@ fn get_api_version(found_attributes: &HashMap<&'static str, Attribute>) -> Resul
 }
 
 /// Generate the declaration of the trait for the client side.
-fn generate_client_side_decls(decls: &[ItemTrait]) -> TokenStream {
+fn generate_client_side_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 	let mut result = Vec::new();
 
 	for decl in decls {
@@ -848,14 +857,12 @@ fn generate_client_side_decls(decls: &[ItemTrait]) -> TokenStream {
 
 		let api_version = get_api_version(&found_attributes);
 
-		let runtime_info = unwrap_or_error(
-			api_version.map(|v| generate_runtime_info_impl(&decl, v))
-		);
+		let runtime_info = api_version.map(|v| generate_runtime_info_impl(&decl, v))?;
 
 		result.push(quote!( #decl #runtime_info #( #errors )* ));
 	}
 
-	quote!( #( #result )* )
+	Ok(quote!( #( #result )* ))
 }
 
 /// Checks that a trait declaration is in the format we expect.
@@ -908,15 +915,17 @@ impl<'ast> Visit<'ast> for CheckTraitDecl {
 }
 
 /// Check that the trait declarations are in the format we expect.
-fn check_trait_decls(decls: &[ItemTrait]) -> Option<TokenStream> {
+fn check_trait_decls(decls: &[ItemTrait]) -> Result<()> {
 	let mut checker = CheckTraitDecl { errors: Vec::new() };
 	decls.iter().for_each(|decl| visit::visit_item_trait(&mut checker, &decl));
 
-	if checker.errors.is_empty() {
-		None
+	if let Some(err) = checker.errors.pop() {
+		Err(checker.errors.into_iter().fold(err, |mut err, other| {
+			err.combine(other);
+			err
+		}))
 	} else {
-		let errors = checker.errors.into_iter().map(|e| e.to_compile_error());
-		Some(quote!( #( #errors )* ))
+		Ok(())
 	}
 }
 
@@ -925,19 +934,23 @@ pub fn decl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::Tok
 	// Parse all trait declarations
 	let RuntimeApiDecls { decls: api_decls } = parse_macro_input!(input as RuntimeApiDecls);
 
-	if let Some(errors) = check_trait_decls(&api_decls) {
-		return errors.into();
-	}
+	decl_runtime_apis_impl_inner(&api_decls).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+fn decl_runtime_apis_impl_inner(api_decls: &[ItemTrait]) -> Result<TokenStream> {
+	check_trait_decls(&api_decls)?;
 
 	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
-	let runtime_decls = generate_runtime_decls(&api_decls);
-	let client_side_decls = generate_client_side_decls(&api_decls);
+	let runtime_decls = generate_runtime_decls(api_decls)?;
+	let client_side_decls = generate_client_side_decls(api_decls)?;
 
-	quote!(
-		#hidden_includes
+	Ok(
+		quote!(
+			#hidden_includes
 
-		#runtime_decls
+			#runtime_decls
 
-		#client_side_decls
-	).into()
+			#client_side_decls
+		)
+	)
 }

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -15,10 +15,11 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::utils::{
-	unwrap_or_error, generate_crate_access, generate_hidden_includes,
+	generate_crate_access, generate_hidden_includes,
 	generate_runtime_mod_name_for_trait, generate_method_runtime_api_impl_name,
 	extract_parameter_names_types_and_borrows, generate_native_call_generator_fn_name,
 	return_type_extract_type, generate_call_api_at_fn_name, prefix_function_with_trait,
+	extract_all_signature_types,
 };
 
 use proc_macro2::{Span, TokenStream};
@@ -28,7 +29,7 @@ use quote::quote;
 use syn::{
 	spanned::Spanned, parse_macro_input, Ident, Type, ItemImpl, Path, Signature,
 	ImplItem, parse::{Parse, ParseStream, Result, Error}, PathArguments, GenericArgument, TypePath,
-	fold::{self, Fold}, parse_quote
+	fold::{self, Fold}, parse_quote,
 };
 
 use std::{collections::HashSet, iter};
@@ -49,7 +50,11 @@ impl Parse for RuntimeApiImpls {
 			impls.push(ItemImpl::parse(input)?);
 		}
 
-		Ok(Self { impls })
+		if impls.is_empty() {
+			Err(Error::new(Span::call_site(), "No api implementation given!"))
+		} else {
+			Ok(Self { impls })
+		}
 	}
 }
 
@@ -222,59 +227,69 @@ fn generate_wasm_interface(impls: &[ItemImpl]) -> Result<TokenStream> {
 	Ok(quote!( #( #impl_calls )* ))
 }
 
-fn generate_block_and_block_id_ty(
-	runtime: &Type,
-	trait_: &'static str,
-	assoc_type: &'static str,
-) -> (TokenStream, TokenStream) {
+fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-	let trait_ = Ident::new(trait_, Span::call_site());
-	let assoc_type = Ident::new(assoc_type, Span::call_site());
-
-	let block = quote!( <#runtime as #crate_::#trait_>::#assoc_type );
-	let block_id = quote!( #crate_::BlockId<#block> );
-
-	(block, block_id)
-}
-
-fn generate_node_block_and_block_id_ty(runtime: &Type) -> (TokenStream, TokenStream) {
-	generate_block_and_block_id_ty(runtime, "GetNodeBlockType", "NodeBlock")
-}
-
-fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStream> {
-	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-	let runtime = &impls.get(0).ok_or_else(||
-		Error::new(Span::call_site(), "No api implementation given!")
-	)?.self_ty;
-	let (block, block_id) = generate_node_block_and_block_id_ty(runtime);
 
 	Ok(quote!(
 		pub struct RuntimeApi {}
 		/// Implements all runtime apis for the client side.
 		#[cfg(any(feature = "std", test))]
-		pub struct RuntimeApiImpl<C: #crate_::CallRuntimeAt<#block> + 'static> {
+		pub struct RuntimeApiImpl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block> + 'static>
+			where
+				// Rust bug: https://github.com/rust-lang/rust/issues/24159
+				C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			call: &'static C,
 			commit_on_success: std::cell::RefCell<bool>,
-			initialized_block: std::cell::RefCell<Option<#block_id>>,
+			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			recorder: Option<#crate_::ProofRecorder<#block>>,
+			storage_transaction_cache: std::cell::RefCell<
+				#crate_::StorageTransactionCache<Block, C::StateBackend>
+			>,
+			recorder: Option<#crate_::ProofRecorder<Block>>,
 		}
 
 		// `RuntimeApi` itself is not threadsafe. However, an instance is only available in a
 		// `ApiRef` object and `ApiRef` also has an associated lifetime. This lifetimes makes it
 		// impossible to move `RuntimeApi` into another thread.
 		#[cfg(any(feature = "std", test))]
-		unsafe impl<C: #crate_::CallRuntimeAt<#block>> Send for RuntimeApiImpl<C> {}
-		#[cfg(any(feature = "std", test))]
-		unsafe impl<C: #crate_::CallRuntimeAt<#block>> Sync for RuntimeApiImpl<C> {}
+		unsafe impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> Send
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block>> #crate_::ApiExt<#block> for RuntimeApiImpl<C> {
+		unsafe impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> Sync
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{}
+
+		#[cfg(any(feature = "std", test))]
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> #crate_::ApiErrorExt
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			type Error = C::Error;
+		}
+
+		#[cfg(any(feature = "std", test))]
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> #crate_::ApiExt<Block> for
+			RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
+			type StateBackend = C::StateBackend;
 
 			fn map_api_result<F: FnOnce(&Self) -> std::result::Result<R, E>, R, E>(
 				&self,
-				map_call: F
+				map_call: F,
 			) -> std::result::Result<R, E> where Self: Sized {
 				*self.commit_on_success.borrow_mut() = false;
 				let res = map_call(self);
@@ -287,7 +302,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 
 			fn runtime_version_at(
 				&self,
-				at: &#block_id
+				at: &#crate_::BlockId<Block>,
 			) -> std::result::Result<#crate_::RuntimeVersion, C::Error> {
 				self.call.runtime_version_at(at)
 			}
@@ -307,13 +322,37 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 						#crate_::StorageProof::new(trie_nodes)
 					})
 			}
+
+			fn into_storage_changes<
+				T: #crate_::ChangesTrieStorage<#crate_::HasherFor<Block>, #crate_::NumberFor<Block>>
+			>(
+				&self,
+				backend: &Self::StateBackend,
+				changes_trie_storage: Option<&T>,
+				parent_hash: Block::Hash,
+			) -> std::result::Result<
+				#crate_::StorageChanges<Self::StateBackend, Block>,
+				String
+			> where Self: Sized {
+				self.initialized_block.borrow_mut().take();
+				self.changes.replace(Default::default()).into_storage_changes(
+					backend,
+					changes_trie_storage,
+					parent_hash,
+					self.storage_transaction_cache.replace(Default::default()),
+				)
+			}
 		}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block> + 'static> #crate_::ConstructRuntimeApi<#block, C>
+		impl<Block: #crate_::BlockT, C> #crate_::ConstructRuntimeApi<Block, C>
 			for RuntimeApi
+				where
+					C: #crate_::CallApiAt<Block> + 'static,
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
 		{
-			type RuntimeApi = RuntimeApiImpl<C>;
+			type RuntimeApi = RuntimeApiImpl<Block, C>;
 
 			fn construct_runtime_api<'a>(
 				call: &'a C,
@@ -324,20 +363,26 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					initialized_block: None.into(),
 					changes: Default::default(),
 					recorder: Default::default(),
+					storage_transaction_cache: Default::default(),
 				}.into()
 			}
 		}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block>> RuntimeApiImpl<C> {
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> RuntimeApiImpl<Block, C>
+			where
+				// Rust bug: https://github.com/rust-lang/rust/issues/24159
+				C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			fn call_api_at<
 				R: #crate_::Encode + #crate_::Decode + PartialEq,
 				F: FnOnce(
 					&C,
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
-					&std::cell::RefCell<Option<#crate_::BlockId<#block>>>,
-					&Option<#crate_::ProofRecorder<#block>>,
+					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
+					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
+					&Option<#crate_::ProofRecorder<Block>>,
 				) -> std::result::Result<#crate_::NativeOrEncoded<R>, E>,
 				E,
 			>(
@@ -348,6 +393,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					&self.call,
 					self,
 					&self.changes,
+					&self.storage_transaction_cache,
 					&self.initialized_block,
 					&self.recorder,
 				);
@@ -413,9 +459,7 @@ fn generate_api_impl_for_runtime(impls: &[ItemImpl]) -> Result<TokenStream> {
 /// `impl Api for Runtime` with `impl Api for RuntimeApi` and replace the method implementations
 /// with code that calls into the runtime.
 struct ApiRuntimeImplToApiRuntimeApiImpl<'a> {
-	node_block: &'a TokenStream,
 	runtime_block: &'a TypePath,
-	node_block_id: &'a TokenStream,
 	runtime_mod_path: &'a Path,
 	runtime_type: &'a Type,
 	trait_generic_arguments: &'a [GenericArgument],
@@ -425,8 +469,7 @@ struct ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	fn fold_type_path(&mut self, input: TypePath) -> TypePath {
 		let new_ty_path = if input == *self.runtime_block {
-			let node_block = self.node_block;
-			parse_quote!( #node_block )
+			parse_quote!( __SR_API_BLOCK__ )
 		} else {
 			input
 		};
@@ -442,9 +485,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 				generate_native_call_generator_fn_name(&input.sig.ident);
 			let call_api_at_call = generate_call_api_at_fn_name(&input.sig.ident);
 			let trait_generic_arguments = self.trait_generic_arguments;
-			let node_block = self.node_block;
 			let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-			let block_id = self.node_block_id;
 
 			// Generate the access to the native parameters
 			let param_tuple_access = if input.sig.inputs.len() == 1 {
@@ -471,7 +512,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 			// Rewrite the input parameters.
 			input.sig.inputs = parse_quote! {
 				&self,
-				at: &#block_id,
+				at: &#crate_::BlockId<__SR_API_BLOCK__>,
 				context: #crate_::ExecutionContext,
 				params: Option<( #( #param_types ),* )>,
 				params_encoded: Vec<u8>,
@@ -495,17 +536,25 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 					#error
 
 					self.call_api_at(
-						|call_runtime_at, core_api, changes, initialized_block, recorder| {
+						|
+							call_runtime_at,
+							core_api,
+							changes,
+							storage_transaction_cache,
+							initialized_block,
+							recorder
+						| {
 							#runtime_mod_path #call_api_at_call(
 								call_runtime_at,
 								core_api,
 								at,
 								params_encoded,
 								changes,
+								storage_transaction_cache,
 								initialized_block,
 								params.map(|p| {
 									#runtime_mod_path #native_call_generator_ident ::
-										<#runtime, #node_block #(, #trait_generic_arguments )*> (
+										<#runtime, __SR_API_BLOCK__ #(, #trait_generic_arguments )*> (
 										#( #param_tuple_access ),*
 									)
 								}),
@@ -526,13 +575,50 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	}
 
 	fn fold_item_impl(&mut self, mut input: ItemImpl) -> ItemImpl {
-		// Implement the trait for the `RuntimeApiImpl`
-		input.self_ty = Box::new(parse_quote!( RuntimeApiImpl<RuntimeApiImplCall> ));
+		// All this `UnwindSafe` magic below here is required for this rust bug:
+		// https://github.com/rust-lang/rust/issues/24159
+		// Before we directly had the final block type and rust could determine that it is unwind
+		// safe, but now we just have a generic parameter `Block`.
 
 		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-		let block = self.node_block;
+
+		// Implement the trait for the `RuntimeApiImpl`
+		input.self_ty = Box::new(
+			parse_quote!( RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall> )
+		);
+
 		input.generics.params.push(
-			parse_quote!( RuntimeApiImplCall: #crate_::CallRuntimeAt<#block> + 'static )
+			parse_quote!(
+				__SR_API_BLOCK__: #crate_::BlockT + std::panic::UnwindSafe +
+					std::panic::RefUnwindSafe
+			)
+		);
+		input.generics.params.push(
+			parse_quote!( RuntimeApiImplCall: #crate_::CallApiAt<__SR_API_BLOCK__> + 'static )
+		);
+
+		let where_clause = input.generics.make_where_clause();
+
+		where_clause.predicates.push(
+			parse_quote! {
+				RuntimeApiImplCall::StateBackend:
+					#crate_::StateBackend<#crate_::HasherFor<__SR_API_BLOCK__>>
+			}
+		);
+
+		// Require that all types used in the function signatures are unwind safe.
+		extract_all_signature_types(&input.items).iter().for_each(|i| {
+			where_clause.predicates.push(
+				parse_quote! {
+					#i: std::panic::UnwindSafe + std::panic::RefUnwindSafe
+				}
+			);
+		});
+
+		where_clause.predicates.push(
+			parse_quote! {
+				__SR_API_BLOCK__::Header: std::panic::UnwindSafe + std::panic::RefUnwindSafe
+			}
 		);
 
 		// The implementation for the `RuntimeApiImpl` is only required when compiling with
@@ -555,7 +641,6 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result<TokenStream>
 			.ok_or_else(|| Error::new(impl_trait_path.span(), "Empty trait path not possible!"))?
 			.clone();
 		let runtime_block = extract_runtime_block_ident(impl_trait_path)?;
-		let (node_block, node_block_id) = generate_node_block_and_block_id_ty(&impl_.self_ty);
 		let runtime_type = &impl_.self_ty;
 		let mut runtime_mod_path = extend_with_runtime_decl_path(impl_trait_path.clone());
 		// remove the trait to get just the module path
@@ -568,8 +653,6 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result<TokenStream>
 
 		let mut visitor = ApiRuntimeImplToApiRuntimeApiImpl {
 			runtime_block,
-			node_block: &node_block,
-			node_block_id: &node_block_id,
 			runtime_mod_path: &runtime_mod_path,
 			runtime_type: &*runtime_type,
 			trait_generic_arguments: &trait_generic_arguments,
@@ -627,31 +710,37 @@ pub fn impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::Tok
 	// Parse all impl blocks
 	let RuntimeApiImpls { impls: api_impls } = parse_macro_input!(input as RuntimeApiImpls);
 
-	let dispatch_impl = unwrap_or_error(generate_dispatch_function(&api_impls));
-	let api_impls_for_runtime = unwrap_or_error(generate_api_impl_for_runtime(&api_impls));
-	let base_runtime_api = unwrap_or_error(generate_runtime_api_base_structures(&api_impls));
-	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
-	let runtime_api_versions = unwrap_or_error(generate_runtime_api_versions(&api_impls));
-	let wasm_interface = unwrap_or_error(generate_wasm_interface(&api_impls));
-	let api_impls_for_runtime_api = unwrap_or_error(generate_api_impl_for_runtime_api(&api_impls));
-
-	quote!(
-		#hidden_includes
-
-		#base_runtime_api
-
-		#api_impls_for_runtime
-
-		#api_impls_for_runtime_api
-
-		#runtime_api_versions
-
-		pub mod api {
-			use super::*;
-
-			#dispatch_impl
-
-			#wasm_interface
-		}
-	).into()
+	impl_runtime_apis_impl_inner(&api_impls).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+fn impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result<TokenStream> {
+	let dispatch_impl = generate_dispatch_function(api_impls)?;
+	let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?;
+	let base_runtime_api = generate_runtime_api_base_structures()?;
+	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
+	let runtime_api_versions = generate_runtime_api_versions(api_impls)?;
+	let wasm_interface = generate_wasm_interface(api_impls)?;
+	let api_impls_for_runtime_api = generate_api_impl_for_runtime_api(api_impls)?;
+
+	Ok(
+		quote!(
+			#hidden_includes
+
+			#base_runtime_api
+
+			#api_impls_for_runtime
+
+			#api_impls_for_runtime_api
+
+			#runtime_api_versions
+
+			pub mod api {
+				use super::*;
+
+				#dispatch_impl
+
+				#wasm_interface
+			}
+		)
+	)
 }

substrate/primitives/api/proc-macro/src/utils.rs

@@ -18,6 +18,7 @@ use proc_macro2::{TokenStream, Span};
 
 use syn::{
 	Result, Ident, Signature, parse_quote, Type, Pat, spanned::Spanned, FnArg, Error, token::And,
+	ImplItem, ReturnType,
 };
 
 use quote::quote;
@@ -26,11 +27,6 @@ use std::env;
 
 use proc_macro_crate::crate_name;
 
-/// Unwrap the given result, if it is an error, `compile_error!` will be generated.
-pub fn unwrap_or_error(res: Result<TokenStream>) -> TokenStream {
-	res.unwrap_or_else(|e| e.to_compile_error())
-}
-
 fn generate_hidden_includes_mod_name(unique_id: &'static str) -> Ident {
 	Ident::new(&format!("sp_api_hidden_includes_{}", unique_id), Span::call_site())
 }
@@ -81,10 +77,10 @@ pub fn generate_method_runtime_api_impl_name(trait_: &Ident, method: &Ident) ->
 }
 
 /// Get the type of a `syn::ReturnType`.
-pub fn return_type_extract_type(rt: &syn::ReturnType) -> Type {
+pub fn return_type_extract_type(rt: &ReturnType) -> Type {
 	match rt {
-		syn::ReturnType::Default => parse_quote!( () ),
-		syn::ReturnType::Type(_, ref ty) => *ty.clone(),
+		ReturnType::Default => parse_quote!( () ),
+		ReturnType::Type(_, ref ty) => *ty.clone(),
 	}
 }
 
@@ -176,3 +172,30 @@ pub fn generate_call_api_at_fn_name(fn_name: &Ident) -> Ident {
 pub fn prefix_function_with_trait<F: ToString>(trait_: &Ident, function: &F) -> String {
 	format!("{}_{}", trait_.to_string(), function.to_string())
 }
+
+/// Extract all types that appear in signatures in the given `ImplItem`'s.
+///
+/// If a type is a reference, the inner type is extracted (without the reference).
+pub fn extract_all_signature_types(items: &[ImplItem]) -> Vec<Type> {
+	items.iter()
+		.filter_map(|i| match i {
+			ImplItem::Method(method) => Some(&method.sig),
+			_ => None,
+		})
+		.map(|sig| {
+			let ret_ty = match &sig.output {
+				ReturnType::Default => None,
+				ReturnType::Type(_, ty) => Some((**ty).clone()),
+			};
+
+			sig.inputs.iter().filter_map(|i| match i {
+				FnArg::Typed(arg) => Some(&arg.ty),
+				_ => None,
+			}).map(|ty| match &**ty {
+				Type::Reference(t) => (*t.elem).clone(),
+				_ => (**ty).clone(),
+			}).chain(ret_ty)
+		})
+		.flatten()
+		.collect()
+}

substrate/primitives/api/src/lib.rs

@@ -26,7 +26,7 @@
 //! functionality that every runtime needs to export.
 //!
 //! Besides the macros and the [`Core`] runtime api, this crates provides the [`Metadata`] runtime
-//! api, the [`ApiExt`] trait, the [`CallRuntimeAt`] trait and the [`ConstructRuntimeApi`] trait.
+//! api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi`] trait.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
@@ -35,18 +35,23 @@ extern crate self as sp_api;
 
 #[doc(hidden)]
 #[cfg(feature = "std")]
-pub use sp_state_machine::{OverlayedChanges, StorageProof};
+pub use sp_state_machine::{
+	OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieStorage,
+};
 #[doc(hidden)]
 #[cfg(feature = "std")]
 pub use sp_core::NativeOrEncoded;
 #[doc(hidden)]
+#[cfg(feature = "std")]
+pub use hash_db::Hasher;
+#[doc(hidden)]
 #[cfg(not(feature = "std"))]
 pub use sp_core::to_substrate_wasm_fn_return_value;
 #[doc(hidden)]
 pub use sp_runtime::{
 	traits::{
-		Block as BlockT, GetNodeBlockType, GetRuntimeBlockType,
-		Header as HeaderT, ApiRef, RuntimeApiInfo, Hash as HashT,
+		Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, HasherFor, NumberFor,
+		Header as HeaderT, Hash as HashT,
 	},
 	generic::BlockId, transaction_validity::TransactionValidity,
 };
@@ -157,8 +162,8 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// ```rust
 /// use sp_version::create_runtime_str;
 /// #
-/// # use sp_runtime::traits::GetNodeBlockType;
-/// # use sp_test_primitives::{Block, Header};
+/// # use sp_runtime::traits::{GetNodeBlockType, Block as BlockT};
+/// # use sp_test_primitives::Block;
 /// #
 /// # /// The declaration of the `Runtime` type and the implementation of the `GetNodeBlockType`
 /// # /// trait are done by the `construct_runtime!` macro in a real runtime.
@@ -187,7 +192,7 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// #           unimplemented!()
 /// #       }
 /// #       fn execute_block(_block: Block) {}
-/// #       fn initialize_block(_header: &Header) {}
+/// #       fn initialize_block(_header: &<Block as BlockT>::Header) {}
 /// #   }
 ///
 ///     impl self::Balance<Block> for Runtime {
@@ -221,27 +226,60 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// ```
 pub use sp_api_proc_macro::impl_runtime_apis;
 
-#[cfg(feature = "std")]
 /// A type that records all accessed trie nodes and generates a proof out of it.
+#[cfg(feature = "std")]
 pub type ProofRecorder<B> = sp_state_machine::ProofRecorder<
 	<<<B as BlockT>::Header as HeaderT>::Hashing as HashT>::Hasher
 >;
 
+/// A type that is used as cache for the storage transactions.
+#[cfg(feature = "std")]
+pub type StorageTransactionCache<Block, Backend> =
+	sp_state_machine::StorageTransactionCache<
+		<Backend as StateBackend<HasherFor<Block>>>::Transaction, HasherFor<Block>, NumberFor<Block>
+	>;
+
+#[cfg(feature = "std")]
+pub type StorageChanges<SBackend, Block> =
+	sp_state_machine::StorageChanges<
+		<SBackend as StateBackend<HasherFor<Block>>>::Transaction,
+		HasherFor<Block>,
+		NumberFor<Block>
+	>;
+
+/// Extract the state backend type for a type that implements `ProvideRuntimeApi`.
+#[cfg(feature = "std")]
+pub type StateBackendFor<P, Block> =
+	<<P as ProvideRuntimeApi<Block>>::Api as ApiExt<Block>>::StateBackend;
+
+/// Extract the state backend transaction type for a type that implements `ProvideRuntimeApi`.
+#[cfg(feature = "std")]
+pub type TransactionFor<P, Block> =
+	<StateBackendFor<P, Block> as StateBackend<HasherFor<Block>>>::Transaction;
+
 /// Something that can be constructed to a runtime api.
 #[cfg(feature = "std")]
-pub trait ConstructRuntimeApi<Block: BlockT, C: CallRuntimeAt<Block>> {
+pub trait ConstructRuntimeApi<Block: BlockT, C: CallApiAt<Block>> {
 	/// The actual runtime api that will be constructed.
-	type RuntimeApi;
+	type RuntimeApi: ApiExt<Block>;
 
 	/// Construct an instance of the runtime api.
 	fn construct_runtime_api<'a>(call: &'a C) -> ApiRef<'a, Self::RuntimeApi>;
 }
 
-/// An extension for the `RuntimeApi`.
+/// Extends the runtime api traits with an associated error type. This trait is given as super
+/// trait to every runtime api trait.
 #[cfg(feature = "std")]
-pub trait ApiExt<Block: BlockT> {
-	/// Error type used by the interface.
+pub trait ApiErrorExt {
+	/// Error type used by the runtime apis.
 	type Error: std::fmt::Debug + From<String>;
+}
+
+/// Extends the runtime api implementation with some common functionality.
+#[cfg(feature = "std")]
+pub trait ApiExt<Block: BlockT>: ApiErrorExt {
+	/// The state backend that is used to store the block states.
+	type StateBackend: StateBackend<HasherFor<Block>>;
 
 	/// The given closure will be called with api instance. Inside the closure any api call is
 	/// allowed. After doing the api call, the closure is allowed to map the `Result` to a
@@ -250,15 +288,15 @@ pub trait ApiExt<Block: BlockT> {
 	/// On `Ok`, the structure commits the changes to an internal buffer.
 	fn map_api_result<F: FnOnce(&Self) -> result::Result<R, E>, R, E>(
 		&self,
-		map_call: F
+		map_call: F,
 	) -> result::Result<R, E> where Self: Sized;
 
 	/// Checks if the given api is implemented and versions match.
 	fn has_api<A: RuntimeApiInfo + ?Sized>(
 		&self,
-		at: &BlockId<Block>
+		at: &BlockId<Block>,
 	) -> Result<bool, Self::Error> where Self: Sized {
-		self.runtime_version_at(at).map(|v| v.has_api::<A>())
+		self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, |v| v == A::VERSION))
 	}
 
 	/// Check if the given api is implemented and the version passes a predicate.
@@ -267,7 +305,7 @@ pub trait ApiExt<Block: BlockT> {
 		at: &BlockId<Block>,
 		pred: P,
 	) -> Result<bool, Self::Error> where Self: Sized {
-		self.runtime_version_at(at).map(|v| v.has_api_with::<A, _>(pred))
+		self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, pred))
 	}
 
 	/// Returns the runtime version at the given block id.
@@ -277,8 +315,22 @@ pub trait ApiExt<Block: BlockT> {
 	fn record_proof(&mut self);
 
 	/// Extract the recorded proof.
+	///
 	/// This stops the proof recording.
+	///
+	/// If `record_proof` was not called before, this will return `None`.
 	fn extract_proof(&mut self) -> Option<StorageProof>;
+
+	/// Convert the api object into the storage changes that were done while executing runtime
+	/// api functions.
+	///
+	/// After executing this function, all collected changes are reset.
+	fn into_storage_changes<T: ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>>(
+		&self,
+		backend: &Self::StateBackend,
+		changes_trie_storage: Option<&T>,
+		parent_hash: Block::Hash,
+	) -> Result<StorageChanges<Self::StateBackend, Block>, String> where Self: Sized;
 }
 
 /// Before calling any runtime api function, the runtime need to be initialized
@@ -301,12 +353,44 @@ pub enum InitializeBlock<'a, Block: BlockT> {
 	Do(&'a RefCell<Option<BlockId<Block>>>),
 }
 
-/// Something that can call into the runtime at a given block.
+/// Parameters for [`CallApiAt::call_api_at`].
 #[cfg(feature = "std")]
-pub trait CallRuntimeAt<Block: BlockT> {
-	/// Error type used by the interface.
+pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HasherFor<Block>>> {
+	/// A reference to something that implements the [`Core`] api.
+	pub core_api: &'a C,
+	/// The block id that determines the state that should be setup when calling the function.
+	pub at: &'a BlockId<Block>,
+	/// The name of the function that should be called.
+	pub function: &'static str,
+	/// An optional native call that calls the `function`. This is an optimization to call into a
+	/// native runtime without requiring to encode/decode the parameters. The native runtime can
+	/// still be called when this value is `None`, we then just fallback to encoding/decoding the
+	/// parameters.
+	pub native_call: Option<NC>,
+	/// The encoded arguments of the function.
+	pub arguments: Vec<u8>,
+	/// The overlayed changes that are on top of the state.
+	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
+	/// The cache for storage transactions.
+	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
+	/// Determines if the function requires that `initialize_block` should be called before calling
+	/// the actual function.
+	pub initialize_block: InitializeBlock<'a, Block>,
+	/// The context this function is executed in.
+	pub context: ExecutionContext,
+	/// The optional proof recorder for recording storage accesses.
+	pub recorder: &'a Option<ProofRecorder<Block>>,
+}
+
+/// Something that can call into the an api at a given block.
+#[cfg(feature = "std")]
+pub trait CallApiAt<Block: BlockT> {
+	/// Error type used by the implementation.
 	type Error: std::fmt::Debug + From<String>;
 
+	/// The state backend that is used to store the block states.
+	type StateBackend: StateBackend<HasherFor<Block>>;
+
 	/// Calls the given api function with the given encoded arguments at the given block and returns
 	/// the encoded result.
 	fn call_api_at<
@@ -316,26 +400,66 @@ pub trait CallRuntimeAt<Block: BlockT> {
 		C: Core<Block, Error = Self::Error>,
 	>(
 		&self,
-		core_api: &C,
-		at: &BlockId<Block>,
-		function: &'static str,
-		args: Vec<u8>,
-		changes: &RefCell<OverlayedChanges>,
-		initialize_block: InitializeBlock<'a, Block>,
-		native_call: Option<NC>,
-		context: ExecutionContext,
-		recorder: &Option<ProofRecorder<Block>>,
+		params: CallApiAtParams<'a, Block, C, NC, Self::StateBackend>,
 	) -> Result<NativeOrEncoded<R>, Self::Error>;
 
 	/// Returns the runtime version at the given block.
 	fn runtime_version_at(&self, at: &BlockId<Block>) -> Result<RuntimeVersion, Self::Error>;
 }
 
+/// Auxiliary wrapper that holds an api instance and binds it to the given lifetime.
+#[cfg(feature = "std")]
+pub struct ApiRef<'a, T>(T, std::marker::PhantomData<&'a ()>);
+
+#[cfg(feature = "std")]
+impl<'a, T> From<T> for ApiRef<'a, T> {
+	fn from(api: T) -> Self {
+		ApiRef(api, Default::default())
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'a, T> std::ops::Deref for ApiRef<'a, T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'a, T> std::ops::DerefMut for ApiRef<'a, T> {
+	fn deref_mut(&mut self) -> &mut T {
+		&mut self.0
+	}
+}
+
+/// Something that provides a runtime api.
+#[cfg(feature = "std")]
+pub trait ProvideRuntimeApi<Block: BlockT> {
+	/// The concrete type that provides the api.
+	type Api: ApiExt<Block>;
+
+	/// Returns the runtime api.
+	/// The returned instance will keep track of modifications to the storage. Any successful
+	/// call to an api function, will `commit` its changes to an internal buffer. Otherwise,
+	/// the modifications will be `discarded`. The modifications will not be applied to the
+	/// storage, even on a `commit`.
+	fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api>;
+}
+
+/// Something that provides information about a runtime api.
+#[cfg(feature = "std")]
+pub trait RuntimeApiInfo {
+	/// The identifier of the runtime api.
+	const ID: [u8; 8];
+	/// The version of the runtime api.
+	const VERSION: u32;
+}
+
 /// Extracts the `Api::Error` for a type that provides a runtime api.
 #[cfg(feature = "std")]
-pub type ApiErrorFor<T, Block> = <
-	<T as sp_runtime::traits::ProvideRuntimeApi>::Api as ApiExt<Block>
->::Error;
+pub type ApiErrorFor<T, Block> = <<T as ProvideRuntimeApi<Block>>::Api as ApiErrorExt>::Error;
 
 decl_runtime_apis! {
 	/// The `Core` runtime api that every Substrate runtime needs to implement.

substrate/primitives/api/test/benches/bench.rs

@@ -19,8 +19,9 @@ use substrate_test_runtime_client::{
 	DefaultTestClientBuilderExt, TestClientBuilder,
 	TestClientBuilderExt, runtime::TestAPI,
 };
-use sp_runtime::{generic::BlockId, traits::ProvideRuntimeApi};
+use sp_runtime::generic::BlockId;
 use sp_state_machine::ExecutionStrategy;
+use sp_api::ProvideRuntimeApi;
 
 fn sp_api_benchmark(c: &mut Criterion) {
 	c.bench_function("add one with same runtime api", |b| {

substrate/primitives/api/test/tests/decl_and_impl.rs

@@ -90,16 +90,16 @@ type TestClient = substrate_test_runtime_client::sc_client::Client<
 
 #[test]
 fn test_client_side_function_signature() {
-	let _test: fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>, u64) -> Result<()> =
-		RuntimeApiImpl::<TestClient>::test;
+	let _test: fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>, u64) -> Result<()> =
+		RuntimeApiImpl::<Block, TestClient>::test;
 	let _something_with_block:
-		fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>, Block) -> Result<Block> =
-			RuntimeApiImpl::<TestClient>::something_with_block;
+		fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>, Block) -> Result<Block> =
+			RuntimeApiImpl::<Block, TestClient>::something_with_block;
 
 	#[allow(deprecated)]
 	let _same_name_before_version_2:
-		fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>) -> Result<String> =
-			RuntimeApiImpl::<TestClient>::same_name_before_version_2;
+		fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>) -> Result<String> =
+			RuntimeApiImpl::<Block, TestClient>::same_name_before_version_2;
 }
 
 #[test]

substrate/primitives/api/test/tests/runtime_calls.rs

@@ -14,15 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
+use sp_api::ProvideRuntimeApi;
 use substrate_test_runtime_client::{
 	prelude::*,
 	DefaultTestClientBuilderExt, TestClientBuilder,
 	runtime::{TestAPI, DecodeFails, Transfer, Header},
 };
-use sp_runtime::{
-	generic::BlockId,
-	traits::{ProvideRuntimeApi, Header as HeaderT, Hash as HashT},
-};
+use sp_runtime::{generic::BlockId, traits::{Header as HeaderT, Hash as HashT}};
 use sp_state_machine::{
 	ExecutionStrategy, create_proof_check_backend,
 	execution_proof_check_on_trie_backend,
@@ -174,10 +172,10 @@ fn record_proof_works() {
 
 	// Build the block and record proof
 	let mut builder = client
-		.new_block_at_with_proof_recording(&block_id, Default::default())
+		.new_block_at(&block_id, Default::default(), true)
 		.expect("Creates block builder");
 	builder.push(transaction.clone()).unwrap();
-	let (block, proof) = builder.bake_and_extract_proof().expect("Bake block");
+	let (block, _, proof) = builder.build().expect("Bake block").into_inner();
 
 	let backend = create_proof_check_backend::<<<Header as HeaderT>::Hashing as HashT>::Hasher>(
 		storage_root,

substrate/primitives/api/test/tests/ui/declaring_old_block.stderr

@@ -1,15 +1,15 @@
-error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro!
- --> $DIR/declaring_old_block.rs:4:16
-  |
-4 |     pub trait Api<Block: BlockT> {
-  |                   ^^^^^
-
 error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro! If you try to use a different trait than the substrate `Block` trait, please rename it locally.
  --> $DIR/declaring_old_block.rs:4:23
   |
 4 |     pub trait Api<Block: BlockT> {
   |                          ^^^^^^
 
+error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro!
+ --> $DIR/declaring_old_block.rs:4:16
+  |
+4 |     pub trait Api<Block: BlockT> {
+  |                   ^^^^^
+
 warning: unused import: `sp_runtime::traits::Block as BlockT`
  --> $DIR/declaring_old_block.rs:1:5
   |

substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs

@@ -20,7 +20,7 @@ sp_api::impl_runtime_apis! {
 	}
 
 	impl sp_api::Core<Block> for Runtime {
-		fn version() -> runtime_api::RuntimeVersion {
+		fn version() -> sp_api::RuntimeVersion {
 			unimplemented!()
 		}
 		fn execute_block(_: Block) {

substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr

@@ -1,9 +1,3 @@
-error[E0433]: failed to resolve: use of undeclared type or module `runtime_api`
-  --> $DIR/impl_incorrect_method_signature.rs:23:19
-   |
-23 |         fn version() -> runtime_api::RuntimeVersion {
-   |                         ^^^^^^^^^^^ use of undeclared type or module `runtime_api`
-
 error[E0053]: method `test` has an incompatible type for trait
   --> $DIR/impl_incorrect_method_signature.rs:19:17
    |
@@ -39,8 +33,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr
 33 | | }
    | |_- in this macro invocation
    |
-   = note: expected type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
-              found type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<std::string::String>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
+   = note: expected type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
+              found type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<std::string::String>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
 
 error[E0308]: mismatched types
   --> $DIR/impl_incorrect_method_signature.rs:17:1

substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs

@@ -17,7 +17,7 @@ sp_api::decl_runtime_apis! {
 mod second {
 	use super::*;
 
-	decl_runtime_apis! {
+	sp_api::decl_runtime_apis! {
 		pub trait Api {
 			fn test2(data: u64);
 		}

substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr

@@ -4,74 +4,10 @@ error: Two traits with the same name detected! The trait name is used to generat
 32 |     impl second::Api<Block> for Runtime {
    |                  ^^^
 
-error: cannot find macro `decl_runtime_apis` in this scope
-  --> $DIR/impl_two_traits_with_same_name.rs:20:2
+warning: unused import: `super::*`
+  --> $DIR/impl_two_traits_with_same_name.rs:18:6
    |
-20 |     decl_runtime_apis! {
-   |     ^^^^^^^^^^^^^^^^^
-
-error[E0425]: cannot find function `test2_call_api_at` in `second::runtime_decl_for_Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
+18 |     use super::*;
+   |         ^^^^^^^^
    |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | | ^
-   | | |
-   | |_not found in `second::runtime_decl_for_Api`
-   |   in this macro invocation
-
-error[E0425]: cannot find function `test2_native_call_generator` in `second::runtime_decl_for_Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
-   |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | | ^
-   | | |
-   | |_not found in `second::runtime_decl_for_Api`
-   |   in this macro invocation
-
-error[E0576]: cannot find method or associated constant `test2` in `second::runtime_decl_for_Api::Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:33:6
-   |
-33 |         fn test2(data: u64) {}
-   |            ^^^^^ not found in `second::runtime_decl_for_Api::Api`
-
-error[E0603]: module `runtime_decl_for_Api` is private
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
-   |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | |_^
-
-error[E0119]: conflicting implementations of trait `runtime_decl_for_Api::Api<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>` for type `Runtime`:
-  --> $DIR/impl_two_traits_with_same_name.rs:32:2
-   |
-28 |     impl self::Api<Block> for Runtime {
-   |     --------------------------------- first implementation here
-...
-32 |     impl second::Api<Block> for Runtime {
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ conflicting implementation for `Runtime`
-
-error[E0119]: conflicting implementations of trait `Api<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>` for type `RuntimeApiImpl<_>`:
-  --> $DIR/impl_two_traits_with_same_name.rs:32:2
-   |
-28 |     impl self::Api<Block> for Runtime {
-   |     --------------------------------- first implementation here
-...
-32 |     impl second::Api<Block> for Runtime {
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ conflicting implementation for `RuntimeApiImpl<_>`
+   = note: `#[warn(unused_imports)]` on by default

substrate/primitives/api/test/tests/ui/invalid_api_version.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version.rs:2:4
   |

substrate/primitives/api/test/tests/ui/invalid_api_version_2.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version_2.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version("1")]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version_2.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version("1")]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version_2.rs:2:4
   |

substrate/primitives/api/test/tests/ui/invalid_api_version_3.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version_3.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version()]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version_3.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version()]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version_3.rs:2:4
   |

substrate/primitives/api/test/tests/ui/missing_block_generic_parameter.stderr

@@ -3,9 +3,3 @@ error: Missing `Block` generic parameter.
    |
 18 |     impl self::Api for Runtime {
    |                ^^^
-
-error[E0107]: wrong number of type arguments: expected 1, found 0
-  --> $DIR/missing_block_generic_parameter.rs:18:7
-   |
-18 |     impl self::Api for Runtime {
-   |          ^^^^^^^^^ expected 1 type argument

substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs

@@ -22,7 +22,7 @@ sp_api::impl_runtime_apis! {
 	}
 
 	impl sp_api::Core<Block> for Runtime {
-		fn version() -> runtime_api::RuntimeVersion {
+		fn version() -> sp_api::RuntimeVersion {
 			unimplemented!()
 		}
 		fn execute_block(_: Block) {

substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr

@@ -1,9 +1,3 @@
-error[E0433]: failed to resolve: use of undeclared type or module `runtime_api`
-  --> $DIR/type_reference_in_impl_runtime_apis_call.rs:25:19
-   |
-25 |         fn version() -> runtime_api::RuntimeVersion {
-   |                         ^^^^^^^^^^^ use of undeclared type or module `runtime_api`
-
 error[E0053]: method `test` has an incompatible type for trait
   --> $DIR/type_reference_in_impl_runtime_apis_call.rs:19:17
    |
@@ -39,8 +33,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr
 35 | | }
    | |_- in this macro invocation
    |
-   = note: expected type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
-              found type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<&u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
+   = note: expected type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
+              found type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<&u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
 
 error[E0308]: mismatched types
   --> $DIR/type_reference_in_impl_runtime_apis_call.rs:17:1

substrate/primitives/application-crypto/src/ed25519.rs

@@ -54,4 +54,3 @@ impl RuntimePublic for Public {
 		sp_io::crypto::ed25519_verify(&signature, msg.as_ref(), self)
 	}
 }
-

substrate/primitives/application-crypto/src/sr25519.rs

@@ -53,4 +53,4 @@ impl RuntimePublic for Public {
 	fn verify<M: AsRef<[u8]>>(&self, msg: &M, signature: &Self::Signature) -> bool {
 		sp_io::crypto::sr25519_verify(&signature, msg.as_ref(), self)
 	}
-}
+}

substrate/primitives/application-crypto/test/Cargo.toml

@@ -10,4 +10,5 @@ publish = false
 sp-core = { version = "2.0.0", default-features = false, path = "../../core" }
 substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" }
 sp-runtime = { version = "2.0.0", path = "../../runtime" }
+sp-api = { version = "2.0.0", path = "../../api" }
 sp-application-crypto = { version = "2.0.0", path = "../" }

substrate/primitives/application-crypto/test/src/ed25519.rs

@@ -16,12 +16,13 @@
 
 //! Integration tests for ed25519
 
-use sp_runtime::{generic::BlockId, traits::ProvideRuntimeApi};
+use sp_runtime::generic::BlockId;
 use sp_core::{testing::{KeyStore, ED25519}, crypto::Pair};
 use substrate_test_runtime_client::{
 	TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt,
 	runtime::TestAPI,
 };
+use sp_api::ProvideRuntimeApi;
 use sp_application_crypto::ed25519::{AppPair, AppPublic};
 
 #[test]

substrate/primitives/application-crypto/test/src/sr25519.rs

@@ -17,12 +17,13 @@
 //! Integration tests for sr25519
 
 
-use sp_runtime::{generic::BlockId, traits::ProvideRuntimeApi};
+use sp_runtime::generic::BlockId;
 use sp_core::{testing::{KeyStore, SR25519}, crypto::Pair};
 use substrate_test_runtime_client::{
 	TestClientBuilder, DefaultTestClientBuilderExt, TestClientBuilderExt,
 	runtime::TestAPI,
 };
+use sp_api::ProvideRuntimeApi;
 use sp_application_crypto::sr25519::{AppPair, AppPublic};
 
 #[test]

substrate/primitives/consensus/common/Cargo.toml

@@ -11,6 +11,7 @@ libp2p = { version = "0.14.0-alpha.1", default-features = false }
 log = "0.4.8"
 sp-core = { path= "../../core" }
 sp-inherents = { version = "2.0.0", path = "../../inherents" }
+sp-state-machine = { version = "2.0.0", path = "../../../primitives/state-machine" }
 futures = { version = "0.3.1", features = ["thread-pool"] }
 futures-timer = "0.4.0"
 sp-std = { version = "2.0.0", path = "../../std" }

substrate/primitives/consensus/common/src/block_import.rs

@@ -16,8 +16,10 @@
 
 //! Block import helpers.
 
-use sp_runtime::traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor};
-use sp_runtime::Justification;
+use sp_runtime::{
+	Justification,
+	traits::{Block as BlockT, DigestItemFor, Header as HeaderT, NumberFor, HasherFor},
+};
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -110,7 +112,7 @@ pub struct BlockCheckParams<Block: BlockT> {
 }
 
 /// Data required to import a Block.
-pub struct BlockImportParams<Block: BlockT> {
+pub struct BlockImportParams<Block: BlockT, Transaction> {
 	/// Origin of the Block
 	pub origin: BlockOrigin,
 	/// The header, without consensus post-digests applied. This should be in the same
@@ -130,8 +132,13 @@ pub struct BlockImportParams<Block: BlockT> {
 	/// Digest items that have been added after the runtime for external
 	/// work, like a consensus signature.
 	pub post_digests: Vec<DigestItemFor<Block>>,
-	/// Block's body
+	/// The body of the block.
 	pub body: Option<Vec<Block::Extrinsic>>,
+	/// The changes to the storage to create the state for the block. If this is `Some(_)`,
+	/// the block import will not need to re-execute the block for importing it.
+	pub storage_changes: Option<
+		sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>
+	>,
 	/// Is this block finalized already?
 	/// `true` implies instant finality.
 	pub finalized: bool,
@@ -148,7 +155,7 @@ pub struct BlockImportParams<Block: BlockT> {
 	pub import_existing: bool,
 }
 
-impl<Block: BlockT> BlockImportParams<Block> {
+impl<Block: BlockT, Transaction> BlockImportParams<Block, Transaction> {
 	/// Deconstruct the justified header into parts.
 	pub fn into_inner(self)
 		-> (
@@ -156,7 +163,8 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			<Block as BlockT>::Header,
 			Option<Justification>,
 			Vec<DigestItemFor<Block>>,
-			Option<Vec<<Block as BlockT>::Extrinsic>>,
+			Option<Vec<Block::Extrinsic>>,
+			Option<sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>>,
 			bool,
 			Vec<(Vec<u8>, Option<Vec<u8>>)>,
 		) {
@@ -166,6 +174,7 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			self.justification,
 			self.post_digests,
 			self.body,
+			self.storage_changes,
 			self.finalized,
 			self.auxiliary,
 		)
@@ -186,11 +195,34 @@ impl<Block: BlockT> BlockImportParams<Block> {
 			})
 		}
 	}
+
+	/// Auxiliary function for "converting" the transaction type.
+	///
+	/// Actually this just sets `storage_changes` to `None` and makes rustc think that `Self` now
+	/// uses a different transaction type.
+	pub fn convert_transaction<Transaction2>(self) -> BlockImportParams<Block, Transaction2> {
+		BlockImportParams {
+			origin: self.origin,
+			header: self.header,
+			justification: self.justification,
+			post_digests: self.post_digests,
+			body: self.body,
+			storage_changes: None,
+			finalized: self.finalized,
+			auxiliary: self.auxiliary,
+			allow_missing_state: self.allow_missing_state,
+			fork_choice: self.fork_choice,
+			import_existing: self.import_existing,
+		}
+	}
 }
 
 /// Block import trait.
 pub trait BlockImport<B: BlockT> {
-	type Error: ::std::error::Error + Send + 'static;
+	/// The error type.
+	type Error: std::error::Error + Send + 'static;
+	/// The transaction type used by the backend.
+	type Transaction;
 
 	/// Check block preconditions.
 	fn check_block(
@@ -203,13 +235,14 @@ pub trait BlockImport<B: BlockT> {
 	/// Cached data can be accessed through the blockchain cache.
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Self::Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error>;
 }
 
-impl<B: BlockT> BlockImport<B> for crate::import_queue::BoxBlockImport<B> {
+impl<B: BlockT, Transaction> BlockImport<B> for crate::import_queue::BoxBlockImport<B, Transaction> {
 	type Error = crate::error::Error;
+	type Transaction = Transaction;
 
 	/// Check block preconditions.
 	fn check_block(
@@ -224,17 +257,18 @@ impl<B: BlockT> BlockImport<B> for crate::import_queue::BoxBlockImport<B> {
 	/// Cached data can be accessed through the blockchain cache.
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
 		(**self).import_block(block, cache)
 	}
 }
 
-impl<B: BlockT, T, E: std::error::Error + Send + 'static> BlockImport<B> for Arc<T>
-where for<'r> &'r T: BlockImport<B, Error = E>
+impl<B: BlockT, T, E: std::error::Error + Send + 'static, Transaction> BlockImport<B> for Arc<T>
+	where for<'r> &'r T: BlockImport<B, Error = E, Transaction = Transaction>
 {
 	type Error = E;
+	type Transaction = Transaction;
 
 	fn check_block(
 		&mut self,
@@ -245,7 +279,7 @@ where for<'r> &'r T: BlockImport<B, Error = E>
 
 	fn import_block(
 		&mut self,
-		block: BlockImportParams<B>,
+		block: BlockImportParams<B, Transaction>,
 		cache: HashMap<CacheKeyId, Vec<u8>>,
 	) -> Result<ImportResult, Self::Error> {
 		(&**self).import_block(block, cache)
@@ -254,7 +288,7 @@ where for<'r> &'r T: BlockImport<B, Error = E>
 
 /// Justification import trait
 pub trait JustificationImport<B: BlockT> {
-	type Error: ::std::error::Error + Send + 'static;
+	type Error: std::error::Error + Send + 'static;
 
 	/// Called by the import queue when it is started. Returns a list of justifications to request
 	/// from the network.

substrate/primitives/consensus/common/src/import_queue.rs

@@ -39,13 +39,17 @@ mod basic_queue;
 pub mod buffered_link;
 
 /// Shared block import struct used by the queue.
-pub type BoxBlockImport<B> = Box<dyn BlockImport<B, Error = ConsensusError> + Send + Sync>;
+pub type BoxBlockImport<B, Transaction> = Box<
+	dyn BlockImport<B, Error = ConsensusError, Transaction = Transaction> + Send + Sync
+>;
 
 /// Shared justification import struct used by the queue.
 pub type BoxJustificationImport<B> = Box<dyn JustificationImport<B, Error=ConsensusError> + Send + Sync>;
 
 /// Shared finality proof import struct used by the queue.
-pub type BoxFinalityProofImport<B> = Box<dyn FinalityProofImport<B, Error=ConsensusError> + Send + Sync>;
+pub type BoxFinalityProofImport<B> = Box<
+	dyn FinalityProofImport<B, Error = ConsensusError> + Send + Sync
+>;
 
 /// Maps to the Origin used by the network.
 pub type Origin = libp2p::PeerId;
@@ -83,7 +87,7 @@ pub trait Verifier<B: BlockT>: Send + Sync {
 		header: B::Header,
 		justification: Option<Justification>,
 		body: Option<Vec<B::Extrinsic>>,
-	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String>;
+	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String>;
 }
 
 /// Blocks import queue API.
@@ -176,8 +180,8 @@ pub enum BlockImportError {
 }
 
 /// Single block import function.
-pub fn import_single_block<B: BlockT, V: Verifier<B>>(
-	import_handle: &mut dyn BlockImport<B, Error = ConsensusError>,
+pub fn import_single_block<B: BlockT, V: Verifier<B>, Transaction>(
+	import_handle: &mut dyn BlockImport<B, Transaction = Transaction, Error = ConsensusError>,
 	block_origin: BlockOrigin,
 	block: IncomingBlock<B>,
 	verifier: &mut V,
@@ -254,5 +258,5 @@ pub fn import_single_block<B: BlockT, V: Verifier<B>>(
 	}
 	import_block.allow_missing_state = block.allow_missing_state;
 
-	import_error(import_handle.import_block(import_block, cache))
+	import_error(import_handle.import_block(import_block.convert_transaction(), cache))
 }

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{mem, pin::Pin, time::Duration};
+use std::{mem, pin::Pin, time::Duration, marker::PhantomData};
 use futures::{prelude::*, channel::mpsc, task::Context, task::Poll};
 use futures_timer::Delay;
 use sp_runtime::{Justification, traits::{Block as BlockT, Header as HeaderT, NumberFor}};
@@ -29,7 +29,7 @@ use crate::import_queue::{
 
 /// Interface to a basic block import queue that is importing blocks sequentially in a separate
 /// task, with pluggable verification.
-pub struct BasicQueue<B: BlockT> {
+pub struct BasicQueue<B: BlockT, Transaction> {
 	/// Channel to send messages to the background task.
 	sender: mpsc::UnboundedSender<ToWorkerMsg<B>>,
 	/// Results coming from the worker task.
@@ -40,16 +40,17 @@ pub struct BasicQueue<B: BlockT> {
 	manual_poll: Option<Pin<Box<dyn Future<Output = ()> + Send>>>,
 	/// A thread pool where the background worker is being run.
 	pool: Option<futures::executor::ThreadPool>,
+	_phantom: PhantomData<Transaction>,
 }
 
-impl<B: BlockT> BasicQueue<B> {
+impl<B: BlockT, Transaction: Send + 'static> BasicQueue<B, Transaction> {
 	/// Instantiate a new basic queue, with given verifier.
 	///
 	/// This creates a background task, and calls `on_start` on the justification importer and
 	/// finality proof importer.
 	pub fn new<V: 'static + Verifier<B>>(
 		verifier: V,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		justification_import: Option<BoxJustificationImport<B>>,
 		finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	) -> Self {
@@ -81,11 +82,12 @@ impl<B: BlockT> BasicQueue<B> {
 			result_port,
 			manual_poll,
 			pool,
+			_phantom: PhantomData,
 		}
 	}
 }
 
-impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
+impl<B: BlockT, Transaction: Send> ImportQueue<B> for BasicQueue<B, Transaction> {
 	fn import_blocks(&mut self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>) {
 		if blocks.is_empty() {
 			return;
@@ -102,12 +104,24 @@ impl<B: BlockT> ImportQueue<B> for BasicQueue<B> {
 		number: NumberFor<B>,
 		justification: Justification
 	) {
-		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification));
+		let _ = self.sender
+			.unbounded_send(
+				ToWorkerMsg::ImportJustification(who.clone(), hash, number, justification)
+			);
 	}
 
-	fn import_finality_proof(&mut self, who: Origin, hash: B::Hash, number: NumberFor<B>, finality_proof: Vec<u8>) {
+	fn import_finality_proof(
+		&mut self,
+		who: Origin,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		finality_proof: Vec<u8>,
+	) {
 		trace!(target: "sync", "Scheduling finality proof of {}/{} for import", number, hash);
-		let _ = self.sender.unbounded_send(ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof));
+		let _ = self.sender
+			.unbounded_send(
+				ToWorkerMsg::ImportFinalityProof(who, hash, number, finality_proof)
+			);
 	}
 
 	fn poll_actions(&mut self, cx: &mut Context, link: &mut dyn Link<B>) {
@@ -132,18 +146,19 @@ enum ToWorkerMsg<B: BlockT> {
 	ImportFinalityProof(Origin, B::Hash, NumberFor<B>, Vec<u8>),
 }
 
-struct BlockImportWorker<B: BlockT> {
+struct BlockImportWorker<B: BlockT, Transaction> {
 	result_sender: BufferedLinkSender<B>,
 	justification_import: Option<BoxJustificationImport<B>>,
 	finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	delay_between_blocks: Duration,
+	_phantom: PhantomData<Transaction>,
 }
 
-impl<B: BlockT> BlockImportWorker<B> {
+impl<B: BlockT, Transaction: Send> BlockImportWorker<B, Transaction> {
 	fn new<V: 'static + Verifier<B>>(
 		result_sender: BufferedLinkSender<B>,
 		verifier: V,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		justification_import: Option<BoxJustificationImport<B>>,
 		finality_proof_import: Option<BoxFinalityProofImport<B>>,
 	) -> (impl Future<Output = ()> + Send, mpsc::UnboundedSender<ToWorkerMsg<B>>) {
@@ -154,6 +169,7 @@ impl<B: BlockT> BlockImportWorker<B> {
 			justification_import,
 			finality_proof_import,
 			delay_between_blocks: Duration::new(0, 0),
+			_phantom: PhantomData,
 		};
 
 		// Let's initialize `justification_import` and `finality_proof_import`.
@@ -237,11 +253,11 @@ impl<B: BlockT> BlockImportWorker<B> {
 	/// yielded back in the output once the import is finished.
 	fn import_a_batch_of_blocks<V: 'static + Verifier<B>>(
 		&mut self,
-		block_import: BoxBlockImport<B>,
+		block_import: BoxBlockImport<B, Transaction>,
 		verifier: V,
 		origin: BlockOrigin,
 		blocks: Vec<IncomingBlock<B>>
-	) -> impl Future<Output = (BoxBlockImport<B>, V)> {
+	) -> impl Future<Output = (BoxBlockImport<B, Transaction>, V)> {
 		let mut result_sender = self.result_sender.clone();
 
 		import_many_blocks(block_import, origin, blocks, verifier, self.delay_between_blocks)
@@ -309,16 +325,22 @@ impl<B: BlockT> BlockImportWorker<B> {
 ///
 /// The returned `Future` yields at every imported block, which makes the execution more
 /// fine-grained and making it possible to interrupt the process.
-fn import_many_blocks<B: BlockT, V: Verifier<B>>(
-	import_handle: BoxBlockImport<B>,
+fn import_many_blocks<B: BlockT, V: Verifier<B>, Transaction>(
+	import_handle: BoxBlockImport<B, Transaction>,
 	blocks_origin: BlockOrigin,
 	blocks: Vec<IncomingBlock<B>>,
 	verifier: V,
 	delay_between_blocks: Duration,
-) -> impl Future<Output = (usize, usize, Vec<(
-	Result<BlockImportResult<NumberFor<B>>, BlockImportError>,
-	B::Hash,
-)>, BoxBlockImport<B>, V)> {
+) -> impl Future<
+	Output = (
+		usize,
+		usize,
+		Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash,)>,
+		BoxBlockImport<B, Transaction>,
+		V
+	)
+>
+{
 	let count = blocks.len();
 
 	let blocks_range = match (

substrate/primitives/consensus/common/src/lib.rs

@@ -31,7 +31,9 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use sp_runtime::{traits::{Block as BlockT, DigestFor}, generic::BlockId};
+use sp_runtime::{
+	generic::BlockId, traits::{Block as BlockT, DigestFor, NumberFor, HasherFor},
+};
 use futures::prelude::*;
 pub use sp_inherents::InherentData;
 
@@ -48,10 +50,11 @@ const MAX_BLOCK_SIZE: usize = 4 * 1024 * 1024 + 512;
 
 pub use self::error::Error;
 pub use block_import::{
-	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, BlockCheckParams, ImportResult,
-	JustificationImport, FinalityProofImport,
+	BlockImport, BlockOrigin, ForkChoiceStrategy, ImportedAux, BlockImportParams, BlockCheckParams,
+	ImportResult, JustificationImport, FinalityProofImport,
 };
 pub use select_chain::SelectChain;
+pub use sp_state_machine::Backend as StateBackend;
 
 /// Block status.
 #[derive(Debug, PartialEq, Eq)]
@@ -71,14 +74,56 @@ pub enum BlockStatus {
 /// Environment producer for a Consensus instance. Creates proposer instance and communication streams.
 pub trait Environment<B: BlockT> {
 	/// The proposer type this creates.
-	type Proposer: Proposer<B>;
+	type Proposer: Proposer<B> + 'static;
 	/// Error which can occur upon creation.
-	type Error: From<Error>;
+	type Error: From<Error> + std::fmt::Debug + 'static;
 
 	/// Initialize the proposal logic on top of a specific header. Provide
 	/// the authorities at that header.
-	fn init(&mut self, parent_header: &B::Header)
-		-> Result<Self::Proposer, Self::Error>;
+	fn init(&mut self, parent_header: &B::Header) -> Result<Self::Proposer, Self::Error>;
+}
+
+/// A proposal that is created by a [`Proposer`].
+pub struct Proposal<Block: BlockT, Transaction> {
+	/// The block that was build.
+	pub block: Block,
+	/// Optional proof that was recorded while building the block.
+	pub proof: Option<sp_state_machine::StorageProof>,
+	/// The storage changes while building this block.
+	pub storage_changes: sp_state_machine::StorageChanges<Transaction, HasherFor<Block>, NumberFor<Block>>,
+}
+
+/// Used as parameter to [`Proposer`] to tell the requirement on recording a proof.
+///
+/// When `RecordProof::Yes` is given, all accessed trie nodes should be saved. These recorded
+/// trie nodes can be used by a third party to proof this proposal without having access to the
+/// full storage.
+#[derive(Copy, Clone, PartialEq)]
+pub enum RecordProof {
+	/// `Yes`, record a proof.
+	Yes,
+	/// `No`, don't record any proof.
+	No,
+}
+
+impl RecordProof {
+	/// Returns if `Self` == `Yes`.
+	pub fn yes(&self) -> bool {
+		match self {
+			Self::Yes => true,
+			Self::No => false,
+		}
+	}
+}
+
+impl From<bool> for RecordProof {
+	fn from(val: bool) -> Self {
+		if val {
+			Self::Yes
+		} else {
+			Self::No
+		}
+	}
 }
 
 /// Logic for a proposer.
@@ -89,16 +134,29 @@ pub trait Environment<B: BlockT> {
 /// Proposers are generic over bits of "consensus data" which are engine-specific.
 pub trait Proposer<B: BlockT> {
 	/// Error type which can occur when proposing or evaluating.
-	type Error: From<Error> + ::std::fmt::Debug + 'static;
-	/// Future that resolves to a committed proposal.
-	type Create: Future<Output = Result<B, Self::Error>>;
+	type Error: From<Error> + std::fmt::Debug + 'static;
+	/// The transaction type used by the backend.
+	type Transaction: Default + Send + 'static;
+	/// Future that resolves to a committed proposal with an optional proof.
+	type Proposal: Future<Output = Result<Proposal<B, Self::Transaction>, Self::Error>> +
+		Send + Unpin + 'static;
+
 	/// Create a proposal.
+	///
+	/// Gets the `inherent_data` and `inherent_digests` as input for the proposal. Additionally
+	/// a maximum duration for building this proposal is given. If building the proposal takes
+	/// longer than this maximum, the proposal will be very likely discarded.
+	///
+	/// # Return
+	///
+	/// Returns a future that resolves to a [`Proposal`] or to [`Self::Error`].
 	fn propose(
 		&mut self,
 		inherent_data: InherentData,
 		inherent_digests: DigestFor<B>,
 		max_duration: Duration,
-	) -> Self::Create;
+		record_proof: RecordProof,
+	) -> Self::Proposal;
 }
 
 /// An oracle for when major synchronization work is being undertaken.

substrate/primitives/io/src/lib.rs

@@ -290,7 +290,8 @@ pub trait Storage {
 	///
 	/// Returns an `Option` that holds the SCALE encoded hash.
 	fn changes_root(&mut self, parent_hash: &[u8]) -> Option<Vec<u8>> {
-		self.storage_changes_root(parent_hash).ok().and_then(|h| h)
+		self.storage_changes_root(parent_hash)
+			.expect("Invalid `parent_hash` given to `changes_root`.")
 	}
 
 	/// Get the next key in storage after the given one in lexicographic order.

substrate/primitives/runtime-interface/test/src/lib.rs

@@ -17,6 +17,8 @@
 //! Integration tests for runtime interface primitives
 #![cfg(test)]
 
+#![cfg(test)]
+
 use sp_runtime_interface::*;
 use sp_runtime_interface_test_wasm::{WASM_BINARY, test_api::HostFunctions};
 use sp_wasm_interface::HostFunctions as HostFunctionsT;

substrate/primitives/runtime/src/generic/header.rs

@@ -107,7 +107,7 @@ impl<Number, Hash> traits::Header for Header<Number, Hash> where
 	Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + MaybeDisplay +
 		SimpleArithmetic + Codec + Copy + Into<U256> + TryFrom<U256> + sp_std::str::FromStr,
 	Hash: HashT,
-	Hash::Output: Default + sp_std::hash::Hash + Copy + Member +
+	Hash::Output: Default + sp_std::hash::Hash + Copy + Member + Ord +
 		MaybeSerialize + Debug + MaybeDisplay + SimpleBitOps + Codec,
 {
 	type Number = Number;

substrate/primitives/runtime/src/traits.rs

@@ -526,7 +526,7 @@ pub trait Header: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 's
 	type Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash
 		+ Copy + MaybeDisplay + SimpleArithmetic + Codec + sp_std::str::FromStr;
 	/// Header hash type
-	type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash
+	type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + Ord
 		+ Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>;
 	/// Hashing algorithm
 	type Hashing: Hash<Output = Self::Hash>;
@@ -581,7 +581,7 @@ pub trait Block: Clone + Send + Sync + Codec + Eq + MaybeSerialize + Debug + 'st
 	/// Header type.
 	type Header: Header<Hash=Self::Hash>;
 	/// Block hash type.
-	type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash
+	type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + Ord
 		+ Copy + MaybeDisplay + Default + SimpleBitOps + Codec + AsRef<[u8]> + AsMut<[u8]>;
 
 	/// Returns a reference to the header.
@@ -626,6 +626,8 @@ pub trait Extrinsic: Sized {
 	fn new(_call: Self::Call, _signed_data: Option<Self::SignaturePayload>) -> Option<Self> { None }
 }
 
+/// Extract the hasher type for a block.
+pub type HasherFor<B> = <HashFor<B> as Hash>::Hasher;
 /// Extract the hashing type for a block.
 pub type HashFor<B> = <<B as Block>::Header as Header>::Hashing;
 /// Extract the number type for a block.
@@ -889,42 +891,6 @@ pub trait Applyable: Sized + Send + Sync {
 	) -> crate::ApplyExtrinsicResult;
 }
 
-/// Auxiliary wrapper that holds an api instance and binds it to the given lifetime.
-pub struct ApiRef<'a, T>(T, sp_std::marker::PhantomData<&'a ()>);
-
-impl<'a, T> From<T> for ApiRef<'a, T> {
-	fn from(api: T) -> Self {
-		ApiRef(api, Default::default())
-	}
-}
-
-impl<'a, T> sp_std::ops::Deref for ApiRef<'a, T> {
-	type Target = T;
-
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-impl<'a, T> sp_std::ops::DerefMut for ApiRef<'a, T> {
-	fn deref_mut(&mut self) -> &mut T {
-		&mut self.0
-	}
-}
-
-/// Something that provides a runtime api.
-pub trait ProvideRuntimeApi {
-	/// The concrete type that provides the api.
-	type Api;
-
-	/// Returns the runtime api.
-	/// The returned instance will keep track of modifications to the storage. Any successful
-	/// call to an api function, will `commit` its changes to an internal buffer. Otherwise,
-	/// the modifications will be `discarded`. The modifications will not be applied to the
-	/// storage, even on a `commit`.
-	fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api>;
-}
-
 /// A marker trait for something that knows the type of the runtime block.
 pub trait GetRuntimeBlockType {
 	/// The `RuntimeBlock` type.
@@ -937,14 +903,6 @@ pub trait GetNodeBlockType {
 	type NodeBlock: self::Block;
 }
 
-/// Something that provides information about a runtime api.
-pub trait RuntimeApiInfo {
-	/// The identifier of the runtime api.
-	const ID: [u8; 8];
-	/// The version of the runtime api.
-	const VERSION: u32;
-}
-
 /// Something that can validate unsigned extrinsics for the transaction pool.
 ///
 /// Note that any checks done here are only used for determining the validity of

substrate/primitives/session/src/lib.rs

@@ -21,7 +21,9 @@
 use sp_std::vec::Vec;
 
 #[cfg(feature = "std")]
-use sp_runtime::{generic::BlockId, traits::{ProvideRuntimeApi, Block as BlockT}};
+use sp_runtime::{generic::BlockId, traits::Block as BlockT};
+#[cfg(feature = "std")]
+use sp_api::ProvideRuntimeApi;
 
 sp_api::decl_runtime_apis! {
 	/// Session keys runtime api.
@@ -44,11 +46,11 @@ pub fn generate_initial_session_keys<Block, T>(
 	client: std::sync::Arc<T>,
 	at: &BlockId<Block>,
 	seeds: Vec<String>,
-) -> Result<(), <<T as ProvideRuntimeApi>::Api as sp_api::ApiExt<Block>>::Error>
+) -> Result<(), sp_api::ApiErrorFor<T, Block>>
 where
 	Block: BlockT,
-	T: ProvideRuntimeApi,
-	<T as ProvideRuntimeApi>::Api: SessionKeys<Block>,
+	T: ProvideRuntimeApi<Block>,
+	T::Api: SessionKeys<Block>,
 {
 	let runtime_api = client.runtime_api();
 

substrate/primitives/sr-api/proc-macro/src/lib.rs

@@ -0,0 +1,189 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Macros for declaring and implementing runtime apis.
+
+#![recursion_limit = "512"]
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+
+mod impl_runtime_apis;
+mod decl_runtime_apis;
+mod utils;
+
+/// Tags given trait implementations as runtime apis.
+///
+/// All traits given to this macro, need to be declared with the `decl_runtime_apis!` macro.
+/// The implementation of the trait should follow the declaration given to the `decl_runtime_apis!`
+/// macro, besides the `Block` type that is required as first generic parameter for each runtime
+/// api trait. When implementing a runtime api trait, it is required that the trait is referenced
+/// by a path, e.g. `impl my_trait::MyTrait for Runtime`. The macro will use this path to access
+/// the declaration of the trait for the runtime side.
+///
+/// The macro also generates the api implementations for the client side and provides it through
+/// the `RuntimeApi` type. The `RuntimeApi` is hidden behind a `feature` called `std`.
+///
+/// To expose version information about all implemented api traits, the constant
+/// `RUNTIME_API_VERSIONS` is generated. This constant should be used to instantiate the `apis`
+/// field of `RuntimeVersion`.
+///
+/// # Example
+///
+/// ```rust
+/// use sp_version::create_runtime_str;
+/// #
+/// # use sp_runtime::traits::{GetNodeBlockType, Block as BlockT};
+/// # use test_client::runtime::Block;
+/// #
+/// # /// The declaration of the `Runtime` type and the implementation of the `GetNodeBlockType`
+/// # /// trait are done by the `construct_runtime!` macro in a real runtime.
+/// # pub struct Runtime {}
+/// # impl GetNodeBlockType for Runtime {
+/// #     type NodeBlock = Block;
+/// # }
+/// #
+/// # sp_api::decl_runtime_apis! {
+/// #     /// Declare the api trait.
+/// #     pub trait Balance {
+/// #         /// Get the balance.
+/// #         fn get_balance() -> u64;
+/// #         /// Set the balance.
+/// #         fn set_balance(val: u64);
+/// #     }
+/// #     pub trait BlockBuilder {
+/// #        fn build_block() -> Block;
+/// #    }
+/// # }
+///
+/// /// All runtime api implementations need to be done in one call of the macro!
+/// sp_api::impl_runtime_apis! {
+/// #   impl sp_api::Core<Block> for Runtime {
+/// #       fn version() -> sp_version::RuntimeVersion {
+/// #           unimplemented!()
+/// #       }
+/// #       fn execute_block(_block: Block) {}
+/// #       fn initialize_block(_header: &<Block as BlockT>::Header) {}
+/// #   }
+///
+///     impl self::Balance<Block> for Runtime {
+///         fn get_balance() -> u64 {
+///             1
+///         }
+///         fn set_balance(_bal: u64) {
+///             // Store the balance
+///         }
+///     }
+///
+///     impl self::BlockBuilder<Block> for Runtime {
+///         fn build_block() -> Block {
+///              unimplemented!("Please implement me!")
+///         }
+///     }
+/// }
+///
+/// /// Runtime version. This needs to be declared for each runtime.
+/// pub const VERSION: sp_version::RuntimeVersion = sp_version::RuntimeVersion {
+///     spec_name: create_runtime_str!("node"),
+///     impl_name: create_runtime_str!("test-node"),
+///     authoring_version: 1,
+///     spec_version: 1,
+///     impl_version: 0,
+///     // Here we are exposing the runtime api versions.
+///     apis: RUNTIME_API_VERSIONS,
+/// };
+///
+/// # fn main() {}
+/// ```
+#[proc_macro]
+pub fn impl_runtime_apis(input: TokenStream) -> TokenStream {
+	impl_runtime_apis::impl_runtime_apis_impl(input)
+}
+
+/// Declares given traits as runtime apis.
+///
+/// The macro will create two declarations, one for using on the client side and one for using
+/// on the runtime side. The declaration for the runtime side is hidden in its own module.
+/// The client side declaration gets two extra parameters per function,
+/// `&self` and `at: &BlockId<Block>`. The runtime side declaration will match the given trait
+/// declaration. Besides one exception, the macro adds an extra generic parameter `Block: BlockT`
+/// to the client side and the runtime side. This generic parameter is usable by the user.
+///
+/// For implementing these macros you should use the `impl_runtime_apis!` macro.
+///
+/// # Example
+///
+/// ```rust
+/// sp_api::decl_runtime_apis! {
+///     /// Declare the api trait.
+///     pub trait Balance {
+///         /// Get the balance.
+///         fn get_balance() -> u64;
+///         /// Set the balance.
+///         fn set_balance(val: u64);
+///     }
+///
+///     /// You can declare multiple api traits in one macro call.
+///     /// In one module you can call the macro at maximum one time.
+///     pub trait BlockBuilder {
+///         /// The macro adds an explicit `Block: BlockT` generic parameter for you.
+///         /// You can use this generic parameter as you would defined it manually.
+///         fn build_block() -> Block;
+///     }
+/// }
+///
+/// # fn main() {}
+/// ```
+///
+/// # Runtime api trait versioning
+///
+/// To support versioning of the traits, the macro supports the attribute `#[api_version(1)]`.
+/// The attribute supports any `u32` as version. By default, each trait is at version `1`, if no
+/// version is provided. We also support changing the signature of a method. This signature
+/// change is highlighted with the `#[changed_in(2)]` attribute above a method. A method that is
+/// tagged with this attribute is callable by the name `METHOD_before_version_VERSION`. This
+/// method will only support calling into wasm, trying to call into native will fail (change the
+/// spec version!). Such a method also does not need to be implemented in the runtime.
+///
+/// ```rust
+/// sp_api::decl_runtime_apis! {
+///     /// Declare the api trait.
+///     #[api_version(2)]
+///     pub trait Balance {
+///         /// Get the balance.
+///         fn get_balance() -> u64;
+///         /// Set balance.
+///         fn set_balance(val: u64);
+///         /// Set balance, old version.
+///         ///
+///         /// Is callable by `set_balance_before_version_2`.
+///         #[changed_in(2)]
+///         fn set_balance(val: u16);
+///         /// In version 2, we added this new function.
+///         fn increase_balance(val: u64);
+///     }
+/// }
+///
+/// # fn main() {}
+/// ```
+///
+/// To check if a given runtime implements a runtime api trait, the `RuntimeVersion` has the
+/// function `has_api<A>()`. Also the `ApiExt` provides a function `has_api<A>(at: &BlockId)` to
+/// check if the runtime at the given block id implements the requested runtime api trait.
+#[proc_macro]
+pub fn decl_runtime_apis(input: TokenStream) -> TokenStream {
+	decl_runtime_apis::decl_runtime_apis_impl(input)
+}

substrate/primitives/state-machine/src/backend.rs

@@ -16,17 +16,13 @@
 
 //! State machine backends. These manage the code and storage of contracts.
 
-use std::{error, fmt, cmp::Ord, collections::{HashMap, BTreeMap}, marker::PhantomData, ops};
 use log::warn;
 use hash_db::Hasher;
 use crate::trie_backend::TrieBackend;
 use crate::trie_backend_essence::TrieBackendStorage;
-use sp_trie::{
-	TrieMut, MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration,
-	trie_types::{TrieDBMut, Layout},
-};
-use codec::{Encode, Codec};
-use sp_core::storage::{ChildInfo, OwnedChildInfo, Storage};
+use sp_trie::{TrieMut, MemoryDB, trie_types::TrieDBMut};
+use codec::Encode;
+use sp_core::storage::{ChildInfo, OwnedChildInfo};
 
 /// A state backend is used to read state data and can have changes committed
 /// to it.
@@ -37,7 +33,7 @@ pub trait Backend<H: Hasher>: std::fmt::Debug {
 	type Error: super::Error;
 
 	/// Storage changes to be applied if committing
-	type Transaction: Consolidate + Default;
+	type Transaction: Consolidate + Default + Send;
 
 	/// Type of trie backend storage.
 	type TrieBackendStorage: TrieBackendStorage<H>;
@@ -317,329 +313,6 @@ impl<H: Hasher, KF: sp_trie::KeyFunction<H>> Consolidate for sp_trie::GenericMem
 	}
 }
 
-/// Error impossible.
-// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121
-#[derive(Debug)]
-pub enum Void {}
-
-impl fmt::Display for Void {
-	fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
-		match *self {}
-	}
-}
-
-impl error::Error for Void {
-	fn description(&self) -> &str { "unreachable error" }
-}
-
-/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
-/// tests and proof checking.
-pub struct InMemory<H: Hasher> {
-	inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>,
-	// This field is only needed for returning reference in `as_trie_backend`.
-	trie: Option<TrieBackend<MemoryDB<H>, H>>,
-	_hasher: PhantomData<H>,
-}
-
-impl<H: Hasher> std::fmt::Debug for InMemory<H> {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		write!(f, "InMemory ({} values)", self.inner.len())
-	}
-}
-
-impl<H: Hasher> Default for InMemory<H> {
-	fn default() -> Self {
-		InMemory {
-			inner: Default::default(),
-			trie: None,
-			_hasher: PhantomData,
-		}
-	}
-}
-
-impl<H: Hasher> Clone for InMemory<H> {
-	fn clone(&self) -> Self {
-		InMemory {
-			inner: self.inner.clone(),
-			trie: None,
-			_hasher: PhantomData,
-		}
-	}
-}
-
-impl<H: Hasher> PartialEq for InMemory<H> {
-	fn eq(&self, other: &Self) -> bool {
-		self.inner.eq(&other.inner)
-	}
-}
-
-impl<H: Hasher> InMemory<H> where H::Out: Codec {
-	/// Copy the state, with applied updates
-	pub fn update(&self, changes: <Self as Backend<H>>::Transaction) -> Self {
-		let mut inner = self.inner.clone();
-		for (child_info, key_values) in changes {
-			let entry = inner.entry(child_info).or_default();
-			for (key, val) in key_values {
-				match val {
-					Some(v) => { entry.insert(key, v); },
-					None => { entry.remove(&key); },
-				}
-			}
-		}
-		inner.into()
-	}
-}
-
-impl<H: Hasher> From<HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>> for InMemory<H> {
-	fn from(inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
-		InMemory {
-			inner: inner,
-			trie: None,
-			_hasher: PhantomData,
-		}
-	}
-}
-
-impl<H: Hasher> From<Storage> for InMemory<H> {
-	fn from(inners: Storage) -> Self {
-		let mut inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
-			= inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect();
-		inner.insert(None, inners.top);
-		InMemory {
-			inner: inner,
-			trie: None,
-			_hasher: PhantomData,
-		}
-	}
-}
-
-impl<H: Hasher> From<BTreeMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
-	fn from(inner: BTreeMap<Vec<u8>, Vec<u8>>) -> Self {
-		let mut expanded = HashMap::new();
-		expanded.insert(None, inner);
-		InMemory {
-			inner: expanded,
-			trie: None,
-			_hasher: PhantomData,
-		}
-	}
-}
-
-impl<H: Hasher> From<Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>>
-	for InMemory<H> {
-	fn from(
-		inner: Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
-	) -> Self {
-		let mut expanded: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
-			= HashMap::new();
-		for (child_info, key_values) in inner {
-			let entry = expanded.entry(child_info).or_default();
-			for (key, value) in key_values {
-				if let Some(value) = value {
-					entry.insert(key, value);
-				}
-			}
-		}
-		expanded.into()
-	}
-}
-
-impl<H: Hasher> InMemory<H> {
-	/// child storage key iterator
-	pub fn child_storage_keys(&self) -> impl Iterator<Item=(&[u8], ChildInfo)> {
-		self.inner.iter().filter_map(|item|
-			item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref()))
-		)
-	}
-}
-
-impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
-	type Error = Void;
-	type Transaction = Vec<(
-		Option<(Vec<u8>, OwnedChildInfo)>,
-		Vec<(Vec<u8>, Option<Vec<u8>>)>,
-	)>;
-	type TrieBackendStorage = MemoryDB<H>;
-
-	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
-	}
-
-	fn child_storage(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		key: &[u8],
-	) -> Result<Option<Vec<u8>>, Self::Error> {
-		Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
-			.and_then(|map| map.get(key).map(Clone::clone)))
-	}
-
-	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
-		Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
-	}
-
-	fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
-		let next_key = self.inner.get(&None)
-			.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
-
-		Ok(next_key)
-	}
-
-	fn next_child_storage_key(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		key: &[u8],
-	) -> Result<Option<Vec<u8>>, Self::Error> {
-		let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
-		let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
-			.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
-
-		Ok(next_key)
-	}
-
-	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
-		self.inner.get(&None).map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
-	}
-
-	fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], mut f: F) {
-		self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix))
-			.for_each(|(k, v)| f(k, v)));
-	}
-
-	fn for_keys_in_child_storage<F: FnMut(&[u8])>(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		mut f: F,
-	) {
-		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
-			.map(|map| map.keys().for_each(|k| f(&k)));
-	}
-
-	fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		prefix: &[u8],
-		f: F,
-	) {
-		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
-			.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
-	}
-
-	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
-	where
-		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
-		<H as Hasher>::Out: Ord,
-	{
-		let existing_pairs = self.inner.get(&None)
-			.into_iter()
-			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
-
-		let transaction: Vec<_> = delta.into_iter().collect();
-		let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned())
-			.collect::<HashMap<_, _>>()
-			.into_iter()
-			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
-		);
-
-		let full_transaction = transaction.into_iter().collect();
-
-		(root, vec![(None, full_transaction)])
-	}
-
-	fn child_storage_root<I>(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		delta: I,
-	) -> (H::Out, bool, Self::Transaction)
-	where
-		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
-		H::Out: Ord
-	{
-		let storage_key = storage_key.to_vec();
-		let child_info = Some((storage_key.clone(), child_info.to_owned()));
-
-
-		let existing_pairs = self.inner.get(&child_info)
-			.into_iter()
-			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
-
-		let transaction: Vec<_> = delta.into_iter().collect();
-		let root = child_trie_root::<Layout<H>, _, _, _>(
-			&storage_key,
-			existing_pairs.chain(transaction.iter().cloned())
-				.collect::<HashMap<_, _>>()
-				.into_iter()
-				.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
-		);
-
-		let full_transaction = transaction.into_iter().collect();
-
-		let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
-
-		(root, is_default, vec![(child_info, full_transaction)])
-	}
-
-	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
-		self.inner.get(&None)
-			.into_iter()
-			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
-			.collect()
-	}
-
-	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
-		self.inner.get(&None)
-			.into_iter()
-			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
-			.collect()
-	}
-
-	fn child_keys(
-		&self,
-		storage_key: &[u8],
-		child_info: ChildInfo,
-		prefix: &[u8],
-	) -> Vec<Vec<u8>> {
-		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
-			.into_iter()
-			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
-			.collect()
-	}
-
-	fn as_trie_backend(&mut self)-> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
-		let mut mdb = MemoryDB::default();
-		let mut new_child_roots = Vec::new();
-		let mut root_map = None;
-		for (child_info, map) in &self.inner {
-			if let Some((storage_key, _child_info)) = child_info.as_ref() {
-				// no need to use child_info at this point because we use a MemoryDB for
-				// proof (with PrefixedMemoryDB it would be needed).
-				let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
-				new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
-			} else {
-				root_map = Some(map);
-			}
-		}
-		let root = match root_map {
-			Some(map) => insert_into_memory_db::<H, _>(
-				&mut mdb,
-				map.clone().into_iter().chain(new_child_roots.into_iter()),
-			)?,
-			None => insert_into_memory_db::<H, _>(
-				&mut mdb,
-				new_child_roots.into_iter(),
-			)?,
-		};
-		self.trie = Some(TrieBackend::new(mdb, root));
-		self.trie.as_ref()
-	}
-}
-
 /// Insert input pairs into memory db.
 pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
 	where
@@ -659,25 +332,3 @@ pub(crate) fn insert_into_memory_db<H, I>(mdb: &mut MemoryDB<H>, input: I) -> Op
 
 	Some(root)
 }
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-
-	/// Assert in memory backend with only child trie keys works as trie backend.
-	#[test]
-	fn in_memory_with_child_trie_only() {
-		let storage = InMemory::<sp_core::Blake2Hasher>::default();
-		let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
-		let mut storage = storage.update(
-			vec![(
-				Some((b"1".to_vec(), child_info.clone())),
-				vec![(b"2".to_vec(), Some(b"3".to_vec()))]
-			)]
-		);
-		let trie_backend = storage.as_trie_backend().unwrap();
-		assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(),
-			Some(b"3".to_vec()));
-		assert!(trie_backend.storage(b"1").unwrap().is_some());
-	}
-}

substrate/primitives/state-machine/src/basic.rs

@@ -19,7 +19,7 @@
 use std::{
 	collections::BTreeMap, any::{TypeId, Any}, iter::FromIterator, ops::Bound
 };
-use crate::backend::{Backend, InMemory};
+use crate::{Backend, InMemoryBackend};
 use hash_db::Hasher;
 use sp_trie::{TrieConfiguration, default_child_trie_root};
 use sp_trie::trie_types::Layout;
@@ -288,7 +288,7 @@ impl Externalities for BasicExternalities {
 		if let Some(child) = self.inner.children.get(storage_key.as_ref()) {
 			let delta = child.data.clone().into_iter().map(|(k, v)| (k, Some(v)));
 
-			InMemory::<Blake2Hasher>::default()
+			InMemoryBackend::<Blake2Hasher>::default()
 				.child_storage_root(storage_key.as_ref(), child.child_info.as_ref(), delta).0
 		} else {
 			default_child_trie_root::<Layout<Blake2Hasher>>(storage_key.as_ref())

substrate/primitives/state-machine/src/changes_trie/build.rs

@@ -339,9 +339,9 @@ fn prepare_digest_input<'a, H, Number>(
 mod test {
 	use codec::Encode;
 	use sp_core::Blake2Hasher;
-	use sp_core::storage::well_known_keys::{EXTRINSIC_INDEX};
+	use sp_core::storage::well_known_keys::EXTRINSIC_INDEX;
 	use sp_core::storage::ChildInfo;
-	use crate::backend::InMemory;
+	use crate::InMemoryBackend;
 	use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage};
 	use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData};
 	use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet};
@@ -351,20 +351,20 @@ mod test {
 	const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2");
 
 	fn prepare_for_build(zero: u64) -> (
-		InMemory<Blake2Hasher>,
+		InMemoryBackend<Blake2Hasher>,
 		InMemoryStorage<Blake2Hasher, u64>,
 		OverlayedChanges,
 		Configuration,
 	) {
 		let config = Configuration { digest_interval: 4, digest_levels: 2 };
-		let backend: InMemory<_> = vec![
+		let backend: InMemoryBackend<_> = vec![
 			(vec![100], vec![255]),
 			(vec![101], vec![255]),
 			(vec![102], vec![255]),
 			(vec![103], vec![255]),
 			(vec![104], vec![255]),
 			(vec![105], vec![255]),
-		].into_iter().collect::<::std::collections::BTreeMap<_, _>>().into();
+		].into_iter().collect::<std::collections::BTreeMap<_, _>>().into();
 		let child_trie_key1 = b"1".to_vec();
 		let child_trie_key2 = b"2".to_vec();
 		let storage = InMemoryStorage::with_inputs(vec![

substrate/primitives/state-machine/src/changes_trie/mod.rs

@@ -84,37 +84,37 @@ pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
 /// Requirements for block number that can be used with changes tries.
 pub trait BlockNumber:
 	Send + Sync + 'static +
-	::std::fmt::Display +
+	std::fmt::Display +
 	Clone +
 	From<u32> + TryInto<u32> + One + Zero +
 	PartialEq + Ord +
-	::std::hash::Hash +
-	::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
-	::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
-	::std::ops::Rem<Self, Output=Self> +
-	::std::ops::AddAssign<Self> +
+	std::hash::Hash +
+	std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
+	std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
+	std::ops::Rem<Self, Output=Self> +
+	std::ops::AddAssign<Self> +
 	num_traits::CheckedMul + num_traits::CheckedSub +
 	Decode + Encode
 {}
 
 impl<T> BlockNumber for T where T:
 	Send + Sync + 'static +
-	::std::fmt::Display +
+	std::fmt::Display +
 	Clone +
 	From<u32> + TryInto<u32> + One + Zero +
 	PartialEq + Ord +
-	::std::hash::Hash +
-	::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
-	::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
-	::std::ops::Rem<Self, Output=Self> +
-	::std::ops::AddAssign<Self> +
+	std::hash::Hash +
+	std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
+	std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
+	std::ops::Rem<Self, Output=Self> +
+	std::ops::AddAssign<Self> +
 	num_traits::CheckedMul + num_traits::CheckedSub +
 	Decode + Encode,
 {}
 
 /// Block identifier that could be used to determine fork of this block.
 #[derive(Debug)]
-pub struct AnchorBlockId<Hash: ::std::fmt::Debug, Number: BlockNumber> {
+pub struct AnchorBlockId<Hash: std::fmt::Debug, Number: BlockNumber> {
 	/// Hash of this block.
 	pub hash: Hash,
 	/// Number of this block.
@@ -173,12 +173,14 @@ pub struct ConfigurationRange<'a, N> {
 /// Compute the changes trie root and transaction for given block.
 /// Returns Err(()) if unknown `parent_hash` has been passed.
 /// Returns Ok(None) if there's no data to perform computation.
-/// Panics if background storage returns an error OR if insert to MemoryDB fails.
+/// Panics if background storage returns an error (and `panic_on_storage_error` is `true`) OR
+/// if insert to MemoryDB fails.
 pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
 	backend: &B,
 	storage: Option<&'a S>,
 	changes: &OverlayedChanges,
 	parent_hash: H::Out,
+	panic_on_storage_error: bool,
 ) -> Result<Option<(MemoryDB<H>, H::Out, CacheAction<H::Out, Number>)>, ()>
 	where
 		H::Out: Ord + 'static + Encode,
@@ -188,6 +190,19 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 		_ => return Ok(None),
 	};
 
+	/// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error.
+	fn maybe_panic<R, E: std::fmt::Debug>(
+		res: std::result::Result<R, E>,
+		panic: bool,
+	) -> std::result::Result<R, ()> {
+		res.map(Ok)
+			.unwrap_or_else(|e| if panic {
+				panic!("changes trie: storage access is not allowed to fail within runtime: {:?}", e)
+			} else {
+				Err(())
+			})
+	}
+
 	// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
 	let config = ConfigurationRange {
 		config,
@@ -200,13 +215,16 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 	let block = parent.number.clone() + One::one();
 
 	// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
-	let (input_pairs, child_input_pairs, digest_input_blocks) = prepare_input::<B, H, Number>(
-		backend,
-		storage,
-		config.clone(),
-		changes,
-		&parent,
-	).expect("changes trie: storage access is not allowed to fail within runtime");
+	let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic(
+		prepare_input::<B, H, Number>(
+			backend,
+			storage,
+			config.clone(),
+			changes,
+			&parent,
+		),
+		panic_on_storage_error,
+	)?;
 
 	// prepare cached data
 	let mut cache_action = prepare_cached_build_data(config, block.clone());
@@ -230,8 +248,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 
 				let (key, value) = input_pair.into();
 				not_empty = true;
-				trie.insert(&key, &value)
-					.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+				maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
 			}
 
 			cache_action = cache_action.insert(
@@ -247,8 +264,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 	{
 		let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
 		for (key, value) in child_roots.into_iter().map(Into::into) {
-			trie.insert(&key, &value)
-				.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+			maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
 		}
 
 		let mut storage_changed_keys = HashSet::new();
@@ -260,9 +276,9 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 			}
 
 			let (key, value) = input_pair.into();
-			trie.insert(&key, &value)
-				.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+			maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?;
 		}
+
 		cache_action = cache_action.insert(
 			None,
 			storage_changed_keys,

substrate/primitives/state-machine/src/changes_trie/storage.rs

@@ -38,7 +38,7 @@ pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
 /// Adapter for using changes trie storage as a TrieBackendEssence' storage.
 pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> {
 	storage: &'a dyn Storage<H, Number>,
-	_hasher: ::std::marker::PhantomData<(H, Number)>,
+	_hasher: std::marker::PhantomData<(H, Number)>,
 }
 
 struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {

substrate/primitives/state-machine/src/ext.rs

@@ -17,18 +17,16 @@
 //! Concrete externalities implementation.
 
 use crate::{
-	backend::Backend, OverlayedChanges,
-	changes_trie::{
-		Storage as ChangesTrieStorage, CacheAction as ChangesTrieCacheAction, build_changes_trie,
-	},
+	backend::Backend, OverlayedChanges, StorageTransactionCache,
+	changes_trie::Storage as ChangesTrieStorage,
 };
 
 use hash_db::Hasher;
 use sp_core::{
 	storage::{ChildStorageKey, well_known_keys::is_child_storage_key, ChildInfo},
-	traits::Externalities, hexdisplay::HexDisplay, hash::H256,
+	traits::Externalities, hexdisplay::HexDisplay,
 };
-use sp_trie::{trie_types::Layout, MemoryDB, default_child_trie_root};
+use sp_trie::{trie_types::Layout, default_child_trie_root};
 use sp_externalities::Extensions;
 use codec::{Decode, Encode};
 
@@ -67,23 +65,20 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
 }
 
 /// Wraps a read-only backend, call executor, and current overlayed changes.
-pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
+pub struct Ext<'a, H, N, B, T>
+	where
+		H: Hasher,
+		B: 'a + Backend<H>,
+		N: crate::changes_trie::BlockNumber,
+{
 	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
 	/// The storage backend to read from.
 	backend: &'a B,
-	/// The storage transaction necessary to commit to the backend. Is cached when
-	/// `storage_root` is called and the cache is cleared on every subsequent change.
-	storage_transaction: Option<(B::Transaction, H::Out)>,
+	/// The cache for the storage transactions.
+	storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 	/// Changes trie storage to read from.
 	changes_trie_storage: Option<&'a T>,
-	/// The changes trie transaction necessary to commit to the changes trie backend.
-	/// Set to Some when `storage_changes_root` is called. Could be replaced later
-	/// by calling `storage_changes_root` again => never used as cache.
-	/// This differs from `storage_transaction` behavior, because the moment when
-	/// `storage_changes_root` is called matters + we need to remember additional
-	/// data at this moment (block number).
-	changes_trie_transaction: Option<(MemoryDB<H>, H::Out, ChangesTrieCacheAction<H::Out, N>)>,
 	/// Pseudo-unique id used for tracing.
 	pub id: u16,
 	/// Dummy usage of N arg.
@@ -94,7 +89,8 @@ pub struct Ext<'a, H, N, B, T> where H: Hasher<Out=H256>, B: 'a + Backend<H> {
 
 impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -103,6 +99,7 @@ where
 	/// Create a new `Ext` from overlayed changes and read-only backend
 	pub fn new(
 		overlay: &'a mut OverlayedChanges,
+		storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 		backend: &'a B,
 		changes_trie_storage: Option<&'a T>,
 		extensions: Option<&'a mut Extensions>,
@@ -110,49 +107,27 @@ where
 		Ext {
 			overlay,
 			backend,
-			storage_transaction: None,
 			changes_trie_storage,
-			changes_trie_transaction: None,
+			storage_transaction_cache,
 			id: rand::random(),
 			_phantom: Default::default(),
 			extensions,
 		}
 	}
 
-	/// Get the transaction necessary to update the backend.
-	pub fn transaction(&mut self) -> (
-		(B::Transaction, H256),
-		Option<crate::ChangesTrieTransaction<H, N>>,
-	) {
-		let _ = self.storage_root();
-
-		let (storage_transaction, changes_trie_transaction) = (
-			self.storage_transaction
-				.take()
-				.expect("storage_transaction always set after calling storage root; qed"),
-			self.changes_trie_transaction
-				.take()
-				.map(|(tx, _, cache)| (tx, cache)),
-		);
-
-		(
-			storage_transaction,
-			changes_trie_transaction,
-		)
-	}
-
 	/// Invalidates the currently cached storage root and the db transaction.
 	///
 	/// Called when there are changes that likely will invalidate the storage root.
 	fn mark_dirty(&mut self) {
-		self.storage_transaction = None;
+		self.storage_transaction_cache.reset();
 	}
 }
 
 #[cfg(test)]
 impl<'a, H, N, B, T> Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -173,7 +148,8 @@ where
 
 impl<'a, H, B, T, N> Externalities for Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -295,6 +271,7 @@ where
 			HexDisplay::from(&key),
 			result.as_ref().map(HexDisplay::from),
 		);
+
 		result
 	}
 
@@ -330,8 +307,8 @@ where
 			HexDisplay::from(&key),
 			result,
 		);
-		result
 
+		result
 	}
 
 	fn exists_child_storage(
@@ -499,7 +476,7 @@ where
 
 	fn storage_root(&mut self) -> Vec<u8> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-		if let Some((_, ref root)) = self.storage_transaction {
+		if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root {
 			trace!(target: "state-trace", "{:04x}: Root (cached) {}",
 				self.id,
 				HexDisplay::from(&root.as_ref()),
@@ -507,35 +484,8 @@ where
 			return root.encode();
 		}
 
-		let child_storage_keys = self.overlay.prospective.children.keys()
-				.chain(self.overlay.committed.children.keys());
-		let child_delta_iter = child_storage_keys.map(|storage_key|
-			(
-				storage_key.clone(),
-				self.overlay.committed.children.get(storage_key)
-					.into_iter()
-					.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
-					.chain(
-						self.overlay.prospective.children.get(storage_key)
-							.into_iter()
-							.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
-					),
-				self.overlay.child_info(storage_key).cloned()
-					.expect("child info initialized in either committed or prospective"),
-			)
-		);
-
-
-		// compute and memoize
-		let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
-			.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
-
-		let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter);
-		self.storage_transaction = Some((transaction, root));
-		trace!(target: "state-trace", "{:04x}: Root {}",
-			self.id,
-			HexDisplay::from(&root.as_ref()),
-		);
+		let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache);
+		trace!(target: "state-trace", "{:04x}: Root {}", self.id, HexDisplay::from(&root.as_ref()));
 		root.encode()
 	}
 
@@ -544,7 +494,7 @@ where
 		storage_key: ChildStorageKey,
 	) -> Vec<u8> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-		if self.storage_transaction.is_some() {
+		if self.storage_transaction_cache.transaction_storage_root.is_some() {
 			let root = self
 				.storage(storage_key.as_ref())
 				.and_then(|k| Decode::decode(&mut &k[..]).ok())
@@ -612,35 +562,33 @@ where
 
 	fn storage_changes_root(&mut self, parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
 		let _guard = sp_panic_handler::AbortGuard::force_abort();
-
-		self.changes_trie_transaction = build_changes_trie::<_, T, H, N>(
+		let root = self.overlay.changes_trie_root(
 			self.backend,
 			self.changes_trie_storage.clone(),
-			self.overlay,
-			H256::decode(&mut &parent_hash[..]).map_err(|e|
+			Decode::decode(&mut &parent_hash[..]).map_err(|e|
 				trace!(
 					target: "state-trace",
 					"Failed to decode changes root parent hash: {}",
 					e,
 				)
 			)?,
-		)?;
-		let result = Ok(
-			self.changes_trie_transaction.as_ref().map(|(_, root, _)| root.encode())
+			true,
+			self.storage_transaction_cache,
 		);
 
 		trace!(target: "state-trace", "{:04x}: ChangesRoot({}) {:?}",
 			self.id,
-			HexDisplay::from(&parent_hash.as_ref()),
-			result,
+			HexDisplay::from(&parent_hash),
+			root,
 		);
-		result
+
+		root.map(|r| r.map(|o| o.encode()))
 	}
 }
 
 impl<'a, H, B, T, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B, T>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
 	B: 'a + Backend<H>,
 	T: 'a + ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
@@ -655,16 +603,16 @@ mod tests {
 	use super::*;
 	use hex_literal::hex;
 	use codec::Encode;
-	use sp_core::{Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
+	use sp_core::{H256, Blake2Hasher, storage::well_known_keys::EXTRINSIC_INDEX, map};
 	use crate::{
 		changes_trie::{
 			Configuration as ChangesTrieConfiguration,
 			InMemoryStorage as InMemoryChangesTrieStorage,
-		}, backend::InMemory, overlayed_changes::OverlayedValue,
+		}, InMemoryBackend, overlayed_changes::OverlayedValue,
 	};
 	use sp_core::storage::{Storage, StorageChild};
 
-	type TestBackend = InMemory<Blake2Hasher>;
+	type TestBackend = InMemoryBackend<Blake2Hasher>;
 	type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher, u64>;
 	type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>;
 
@@ -691,27 +639,30 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_none_when_storage_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		overlay.changes_trie_config = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -721,10 +672,11 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
 		let mut overlay = prepare_overlay_with_changes();
+		let mut cache = StorageTransactionCache::default();
 		overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None;
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, Some(&storage), None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
@@ -733,6 +685,7 @@ mod tests {
 
 	#[test]
 	fn next_storage_key_works() {
+		let mut cache = StorageTransactionCache::default();
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_storage(vec![20], None);
 		overlay.set_storage(vec![30], Some(vec![31]));
@@ -745,7 +698,7 @@ mod tests {
 			children: map![]
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -761,7 +714,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_storage(vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -774,7 +727,7 @@ mod tests {
 		const CHILD_UUID_1: &[u8] = b"unique_id_1";
 		const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(CHILD_UUID_1);
 
-
+		let mut cache = StorageTransactionCache::default();
 		let child = || ChildStorageKey::from_slice(CHILD_KEY_1).unwrap();
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![20], None);
@@ -794,7 +747,7 @@ mod tests {
 		}.into();
 
 
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10]));
@@ -810,7 +763,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50]));

substrate/primitives/state-machine/src/in_memory_backend.rs

@@ -0,0 +1,378 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State machine in memory backend.
+
+use crate::{trie_backend::TrieBackend, backend::{Backend, insert_into_memory_db}};
+use std::{error, fmt, collections::{BTreeMap, HashMap}, marker::PhantomData, ops};
+use hash_db::Hasher;
+use sp_trie::{
+	MemoryDB, child_trie_root, default_child_trie_root, TrieConfiguration, trie_types::Layout,
+};
+use codec::Codec;
+use sp_core::storage::{ChildInfo, OwnedChildInfo, Storage};
+
+/// Error impossible.
+// FIXME: use `!` type when stabilized. https://github.com/rust-lang/rust/issues/35121
+#[derive(Debug)]
+pub enum Void {}
+
+impl fmt::Display for Void {
+	fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
+		match *self {}
+	}
+}
+
+impl error::Error for Void {
+	fn description(&self) -> &str { "unreachable error" }
+}
+
+/// In-memory backend. Fully recomputes tries each time `as_trie_backend` is called but useful for
+/// tests and proof checking.
+pub struct InMemory<H: Hasher> {
+	inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>,
+	// This field is only needed for returning reference in `as_trie_backend`.
+	trie: Option<TrieBackend<MemoryDB<H>, H>>,
+	_hasher: PhantomData<H>,
+}
+
+impl<H: Hasher> std::fmt::Debug for InMemory<H> {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "InMemory ({} values)", self.inner.len())
+	}
+}
+
+impl<H: Hasher> Default for InMemory<H> {
+	fn default() -> Self {
+		InMemory {
+			inner: Default::default(),
+			trie: None,
+			_hasher: PhantomData,
+		}
+	}
+}
+
+impl<H: Hasher> Clone for InMemory<H> {
+	fn clone(&self) -> Self {
+		InMemory {
+			inner: self.inner.clone(),
+			trie: None,
+			_hasher: PhantomData,
+		}
+	}
+}
+
+impl<H: Hasher> PartialEq for InMemory<H> {
+	fn eq(&self, other: &Self) -> bool {
+		self.inner.eq(&other.inner)
+	}
+}
+
+impl<H: Hasher> InMemory<H> {
+	/// Copy the state, with applied updates
+	pub fn update<
+		T: IntoIterator<Item = (Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>
+	>(
+		&self,
+		changes: T,
+	) -> Self {
+		let mut inner = self.inner.clone();
+		for (child_info, key_values) in changes.into_iter() {
+			let entry = inner.entry(child_info).or_default();
+			for (key, val) in key_values {
+				match val {
+					Some(v) => { entry.insert(key, v); },
+					None => { entry.remove(&key); },
+				}
+			}
+		}
+		inner.into()
+	}
+}
+
+impl<H: Hasher> From<HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>>
+	for InMemory<H>
+{
+	fn from(inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>) -> Self {
+		InMemory {
+			inner,
+			trie: None,
+			_hasher: PhantomData,
+		}
+	}
+}
+
+impl<H: Hasher> From<Storage> for InMemory<H> {
+	fn from(inners: Storage) -> Self {
+		let mut inner: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
+			= inners.children.into_iter().map(|(k, c)| (Some((k, c.child_info)), c.data)).collect();
+		inner.insert(None, inners.top);
+		InMemory {
+			inner,
+			trie: None,
+			_hasher: PhantomData,
+		}
+	}
+}
+
+impl<H: Hasher> From<BTreeMap<Vec<u8>, Vec<u8>>> for InMemory<H> {
+	fn from(inner: BTreeMap<Vec<u8>, Vec<u8>>) -> Self {
+		let mut expanded = HashMap::new();
+		expanded.insert(None, inner);
+		InMemory {
+			inner: expanded,
+			trie: None,
+			_hasher: PhantomData,
+		}
+	}
+}
+
+impl<H: Hasher> From<Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>>
+	for InMemory<H> {
+	fn from(
+		inner: Vec<(Option<(Vec<u8>, OwnedChildInfo)>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+	) -> Self {
+		let mut expanded: HashMap<Option<(Vec<u8>, OwnedChildInfo)>, BTreeMap<Vec<u8>, Vec<u8>>>
+			= HashMap::new();
+		for (child_info, key_values) in inner {
+			let entry = expanded.entry(child_info).or_default();
+			for (key, value) in key_values {
+				if let Some(value) = value {
+					entry.insert(key, value);
+				}
+			}
+		}
+		expanded.into()
+	}
+}
+
+impl<H: Hasher> InMemory<H> {
+	/// child storage key iterator
+	pub fn child_storage_keys(&self) -> impl Iterator<Item=(&[u8], ChildInfo)> {
+		self.inner.iter().filter_map(|item|
+			item.0.as_ref().map(|v|(&v.0[..], v.1.as_ref()))
+		)
+	}
+}
+
+impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: Codec {
+	type Error = Void;
+	type Transaction = Vec<(
+		Option<(Vec<u8>, OwnedChildInfo)>,
+		Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	)>;
+	type TrieBackendStorage = MemoryDB<H>;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		Ok(self.inner.get(&None).and_then(|map| map.get(key).map(Clone::clone)))
+	}
+
+	fn child_storage(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		key: &[u8],
+	) -> Result<Option<Vec<u8>>, Self::Error> {
+		Ok(self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
+			.and_then(|map| map.get(key).map(Clone::clone)))
+	}
+
+	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
+		Ok(self.inner.get(&None).map(|map| map.get(key).is_some()).unwrap_or(false))
+	}
+
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
+		self.inner.get(&None)
+			.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
+	}
+
+	fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], mut f: F) {
+		self.inner.get(&None).map(|map| map.iter().filter(|(key, _val)| key.starts_with(prefix))
+			.for_each(|(k, v)| f(k, v)));
+	}
+
+	fn for_keys_in_child_storage<F: FnMut(&[u8])>(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		mut f: F,
+	) {
+		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
+			.map(|map| map.keys().for_each(|k| f(&k)));
+	}
+
+	fn for_child_keys_with_prefix<F: FnMut(&[u8])>(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		prefix: &[u8],
+		f: F,
+	) {
+		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
+			.map(|map| map.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f));
+	}
+
+	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		<H as Hasher>::Out: Ord,
+	{
+		let existing_pairs = self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+
+		let transaction: Vec<_> = delta.into_iter().collect();
+		let root = Layout::<H>::trie_root(existing_pairs.chain(transaction.iter().cloned())
+			.collect::<HashMap<_, _>>()
+			.into_iter()
+			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
+		);
+
+		let full_transaction = transaction.into_iter().collect();
+
+		(root, vec![(None, full_transaction)])
+	}
+
+	fn child_storage_root<I>(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		delta: I,
+	) -> (H::Out, bool, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		H::Out: Ord
+	{
+		let storage_key = storage_key.to_vec();
+		let child_info = Some((storage_key.clone(), child_info.to_owned()));
+
+		let existing_pairs = self.inner.get(&child_info)
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+
+		let transaction: Vec<_> = delta.into_iter().collect();
+		let root = child_trie_root::<Layout<H>, _, _, _>(
+			&storage_key,
+			existing_pairs.chain(transaction.iter().cloned())
+				.collect::<HashMap<_, _>>()
+				.into_iter()
+				.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
+		);
+
+		let full_transaction = transaction.into_iter().collect();
+
+		let is_default = root == default_child_trie_root::<Layout<H>>(&storage_key);
+
+		(root, is_default, vec![(child_info, full_transaction)])
+	}
+
+	fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
+		let next_key = self.inner.get(&None)
+			.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
+
+		Ok(next_key)
+	}
+
+	fn next_child_storage_key(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		key: &[u8],
+	) -> Result<Option<Vec<u8>>, Self::Error> {
+		let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded);
+		let next_key = self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
+			.and_then(|map| map.range::<[u8], _>(range).next().map(|(k, _)| k).cloned());
+
+		Ok(next_key)
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
+			.collect()
+	}
+
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+		self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+			.collect()
+	}
+
+	fn child_keys(
+		&self,
+		storage_key: &[u8],
+		child_info: ChildInfo,
+		prefix: &[u8],
+	) -> Vec<Vec<u8>> {
+		self.inner.get(&Some((storage_key.to_vec(), child_info.to_owned())))
+			.into_iter()
+			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+			.collect()
+	}
+
+	fn as_trie_backend(&mut self)-> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
+		let mut mdb = MemoryDB::default();
+		let mut new_child_roots = Vec::new();
+		let mut root_map = None;
+		for (child_info, map) in &self.inner {
+			if let Some((storage_key, _child_info)) = child_info.as_ref() {
+				// no need to use child_info at this point because we use a MemoryDB for
+				// proof (with PrefixedMemoryDB it would be needed).
+				let ch = insert_into_memory_db::<H, _>(&mut mdb, map.clone().into_iter())?;
+				new_child_roots.push((storage_key.clone(), ch.as_ref().into()));
+			} else {
+				root_map = Some(map);
+			}
+		}
+		let root = match root_map {
+			Some(map) => insert_into_memory_db::<H, _>(
+				&mut mdb,
+				map.clone().into_iter().chain(new_child_roots.into_iter()),
+			)?,
+			None => insert_into_memory_db::<H, _>(
+				&mut mdb,
+				new_child_roots.into_iter(),
+			)?,
+		};
+		self.trie = Some(TrieBackend::new(mdb, root));
+		self.trie.as_ref()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	/// Assert in memory backend with only child trie keys works as trie backend.
+	#[test]
+	fn in_memory_with_child_trie_only() {
+		let storage = InMemory::<sp_core::Blake2Hasher>::default();
+		let child_info = OwnedChildInfo::new_default(b"unique_id_1".to_vec());
+		let mut storage = storage.update(
+			vec![(
+				Some((b"1".to_vec(), child_info.clone())),
+				vec![(b"2".to_vec(), Some(b"3".to_vec()))]
+			)]
+		);
+		let trie_backend = storage.as_trie_backend().unwrap();
+		assert_eq!(trie_backend.child_storage(b"1", child_info.as_ref(), b"2").unwrap(),
+			Some(b"3".to_vec()));
+		assert!(trie_backend.storage(b"1").unwrap().is_some());
+	}
+}

substrate/primitives/state-machine/src/lib.rs

@@ -24,12 +24,13 @@ use hash_db::Hasher;
 use codec::{Decode, Encode, Codec};
 use sp_core::{
 	storage::{well_known_keys, ChildInfo}, NativeOrEncoded, NeverNativeValue,
-	traits::CodeExecutor, hexdisplay::HexDisplay, hash::H256,
+	traits::CodeExecutor, hexdisplay::HexDisplay
 };
 use overlayed_changes::OverlayedChangeSet;
 use sp_externalities::Extensions;
 
 pub mod backend;
+mod in_memory_backend;
 mod changes_trie;
 mod error;
 mod ext;
@@ -56,8 +57,9 @@ pub use changes_trie::{
 	key_changes, key_changes_proof, key_changes_proof_check,
 	prune as prune_changes_tries,
 	oldest_non_pruned_trie as oldest_non_pruned_changes_trie,
+	BlockNumber as ChangesTrieBlockNumber,
 };
-pub use overlayed_changes::OverlayedChanges;
+pub use overlayed_changes::{OverlayedChanges, StorageChanges, StorageTransactionCache};
 pub use proving_backend::{
 	create_proof_check_backend, create_proof_check_backend_storage, merge_storage_proofs,
 	ProofRecorder, ProvingBackend, ProvingBackendRecorder, StorageProof,
@@ -65,6 +67,7 @@ pub use proving_backend::{
 pub use trie_backend_essence::{TrieBackendStorage, Storage};
 pub use trie_backend::TrieBackend;
 pub use error::{Error, ExecutionError};
+pub use in_memory_backend::InMemory as InMemoryBackend;
 
 type CallResult<R, E> = Result<NativeOrEncoded<R>, E>;
 
@@ -140,8 +143,10 @@ impl ExecutionStrategy {
 				warn!(
 					"Consensus error between wasm {:?} and native {:?}. Using wasm.",
 					wasm_result,
-					native_result
+					native_result,
 				);
+				warn!("   Native result {:?}", native_result);
+				warn!("   Wasm result {:?}", wasm_result);
 				wasm_result
 			}),
 		}
@@ -164,7 +169,12 @@ fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E
 }
 
 /// The substrate state machine.
-pub struct StateMachine<'a, B, H, N, T, Exec> where H: Hasher<Out=H256>, B: Backend<H> {
+pub struct StateMachine<'a, B, H, N, T, Exec>
+	where
+		H: Hasher,
+		B: Backend<H>,
+		N: ChangesTrieBlockNumber,
+{
 	backend: &'a B,
 	exec: &'a Exec,
 	method: &'a str,
@@ -173,10 +183,12 @@ pub struct StateMachine<'a, B, H, N, T, Exec> where H: Hasher<Out=H256>, B: Back
 	extensions: Extensions,
 	changes_trie_storage: Option<&'a T>,
 	_marker: PhantomData<(H, N)>,
+	storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
 }
 
 impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor,
 	B: Backend<H>,
 	T: ChangesTrieStorage<H, N>,
@@ -201,51 +213,61 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			overlay,
 			changes_trie_storage,
 			_marker: PhantomData,
+			storage_transaction_cache: None,
 		}
 	}
 
+	/// Use given `cache` as storage transaction cache.
+	///
+	/// The cache will be used to cache storage transactions that can be build while executing a
+	/// function in the runtime. For example, when calculating the storage root a transaction is
+	/// build that will be cached.
+	pub fn with_storage_transaction_cache(
+		mut self,
+		cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
+	) -> Self {
+		self.storage_transaction_cache = cache;
+		self
+	}
+
 	/// Execute a call using the given state backend, overlayed changes, and call executor.
-	/// Produces a state-backend-specific "transaction" which can be used to apply the changes
-	/// to the backing store, such as the disk.
 	///
 	/// On an error, no prospective changes are written to the overlay.
 	///
 	/// Note: changes to code will be in place if this call is made again. For running partial
 	/// blocks (e.g. a transaction at a time), ensure a different method is used.
-	pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<
-		(Vec<u8>, (B::Transaction, H::Out), Option<ChangesTrieTransaction<H, N>>),
-		Box<dyn Error>,
-	> {
+	///
+	/// Returns the SCALE encoded result of the executed function.
+	pub fn execute(&mut self, strategy: ExecutionStrategy) -> Result<Vec<u8>, Box<dyn Error>> {
 		// We are not giving a native call and thus we are sure that the result can never be a native
 		// value.
 		self.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 			strategy.get_manager(),
-			true,
 			None,
-		)
-		.map(|(result, storage_tx, changes_tx)| (
-			result.into_encoded(),
-			storage_tx.expect("storage_tx is always computed when compute_tx is true; qed"),
-			changes_tx,
-		))
+		).map(NativeOrEncoded::into_encoded)
 	}
 
 	fn execute_aux<R, NC>(
 		&mut self,
-		compute_tx: bool,
 		use_native: bool,
 		native_call: Option<NC>,
 	) -> (
 		CallResult<R, Exec::Error>,
 		bool,
-		Option<(B::Transaction, H::Out)>,
-		Option<ChangesTrieTransaction<H, N>>,
 	) where
 		R: Decode + Encode + PartialEq,
 		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
 	{
+		let mut cache = StorageTransactionCache::default();
+
+		let cache = match self.storage_transaction_cache.as_mut() {
+			Some(cache) => cache,
+			None => &mut cache,
+		};
+
 		let mut ext = Ext::new(
 			self.overlay,
+			cache,
 			self.backend,
 			self.changes_trie_storage.clone(),
 			Some(&mut self.extensions),
@@ -268,13 +290,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			native_call,
 		);
 
-		let (storage_delta, changes_delta) = if compute_tx {
-			let (storage_delta, changes_delta) = ext.transaction();
-			(Some(storage_delta), changes_delta)
-		} else {
-			(None, None)
-		};
-
 		trace!(
 			target: "state-trace", "{:04x}: Return. Native={:?}, Result={:?}",
 			id,
@@ -282,37 +297,28 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			result,
 		);
 
-		(result, was_native, storage_delta, changes_delta)
+		(result, was_native)
 	}
 
 	fn execute_call_with_both_strategy<Handler, R, NC>(
 		&mut self,
-		compute_tx: bool,
 		mut native_call: Option<NC>,
 		orig_prospective: OverlayedChangeSet,
 		on_consensus_failure: Handler,
-	) -> (
-		CallResult<R, Exec::Error>,
-		Option<(B::Transaction, H::Out)>,
-		Option<ChangesTrieTransaction<H, N>>,
-	) where
-		R: Decode + Encode + PartialEq,
-		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-		Handler: FnOnce(
-			CallResult<R, Exec::Error>,
-			CallResult<R, Exec::Error>,
-		) -> CallResult<R, Exec::Error>
+	) -> CallResult<R, Exec::Error>
+		where
+			R: Decode + Encode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+			Handler: FnOnce(
+				CallResult<R, Exec::Error>,
+				CallResult<R, Exec::Error>,
+			) -> CallResult<R, Exec::Error>
 	{
-		let (result, was_native, storage_delta, changes_delta) = self.execute_aux(
-			compute_tx,
-			true,
-			native_call.take(),
-		);
+		let (result, was_native) = self.execute_aux(true, native_call.take());
 
 		if was_native {
 			self.overlay.prospective = orig_prospective.clone();
-			let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(
-				compute_tx,
+			let (wasm_result, _) = self.execute_aux(
 				false,
 				native_call,
 			);
@@ -321,71 +327,62 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 				&& result.as_ref().ok() == wasm_result.as_ref().ok())
 				|| result.is_err() && wasm_result.is_err()
 			{
-				(result, storage_delta, changes_delta)
+				result
 			} else {
-				(on_consensus_failure(wasm_result, result), wasm_storage_delta, wasm_changes_delta)
+				on_consensus_failure(wasm_result, result)
 			}
 		} else {
-			(result, storage_delta, changes_delta)
+			result
 		}
 	}
 
 	fn execute_call_with_native_else_wasm_strategy<R, NC>(
 		&mut self,
-		compute_tx: bool,
 		mut native_call: Option<NC>,
 		orig_prospective: OverlayedChangeSet,
-	) -> (
-		CallResult<R, Exec::Error>,
-		Option<(B::Transaction, H::Out)>,
-		Option<ChangesTrieTransaction<H, N>>,
-	) where
-		R: Decode + Encode + PartialEq,
-		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	) -> CallResult<R, Exec::Error>
+		where
+			R: Decode + Encode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
 	{
-		let (result, was_native, storage_delta, changes_delta) = self.execute_aux(
-			compute_tx,
+		let (result, was_native) = self.execute_aux(
 			true,
 			native_call.take(),
 		);
 
 		if !was_native || result.is_ok() {
-			(result, storage_delta, changes_delta)
+			result
 		} else {
 			self.overlay.prospective = orig_prospective.clone();
-			let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(
-				compute_tx,
+			let (wasm_result, _) = self.execute_aux(
 				false,
 				native_call,
 			);
-			(wasm_result, wasm_storage_delta, wasm_changes_delta)
+			wasm_result
 		}
 	}
 
 	/// Execute a call using the given state backend, overlayed changes, and call executor.
-	/// Produces a state-backend-specific "transaction" which can be used to apply the changes
-	/// to the backing store, such as the disk.
 	///
 	/// On an error, no prospective changes are written to the overlay.
 	///
 	/// Note: changes to code will be in place if this call is made again. For running partial
 	/// blocks (e.g. a transaction at a time), ensure a different method is used.
+	///
+	/// Returns the result of the executed function either in native reprensentation `R` or
+	/// in SCALE encoded representation.
 	pub fn execute_using_consensus_failure_handler<Handler, R, NC>(
 		&mut self,
 		manager: ExecutionManager<Handler>,
-		compute_tx: bool,
 		mut native_call: Option<NC>,
-	) -> Result<(
-		NativeOrEncoded<R>,
-		Option<(B::Transaction, H::Out)>,
-		Option<ChangesTrieTransaction<H, N>>,
-	), Box<dyn Error>> where
-		R: Decode + Encode + PartialEq,
-		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
-		Handler: FnOnce(
-			CallResult<R, Exec::Error>,
-			CallResult<R, Exec::Error>,
-		) -> CallResult<R, Exec::Error>
+	) -> Result<NativeOrEncoded<R>, Box<dyn Error>>
+		where
+			R: Decode + Encode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+			Handler: FnOnce(
+				CallResult<R, Exec::Error>,
+				CallResult<R, Exec::Error>,
+			) -> CallResult<R, Exec::Error>
 	{
 		// read changes trie configuration. The reason why we're doing it here instead of the
 		// `OverlayedChanges` constructor is that we need proofs for this read as a part of
@@ -405,10 +402,9 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 		let result = {
 			let orig_prospective = self.overlay.prospective.clone();
 
-			let (result, storage_delta, changes_delta) = match manager {
+			match manager {
 				ExecutionManager::Both(on_consensus_failure) => {
 					self.execute_call_with_both_strategy(
-						compute_tx,
 						native_call.take(),
 						orig_prospective,
 						on_consensus_failure,
@@ -416,7 +412,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 				},
 				ExecutionManager::NativeElseWasm => {
 					self.execute_call_with_native_else_wasm_strategy(
-						compute_tx,
 						native_call.take(),
 						orig_prospective,
 					)
@@ -426,15 +421,12 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 						BackendTrustLevel::Trusted => None,
 						BackendTrustLevel::Untrusted => Some(sp_panic_handler::AbortGuard::never_abort()),
 					};
-					let res = self.execute_aux(compute_tx, false, native_call);
-					(res.0, res.2, res.3)
+					self.execute_aux(false, native_call).0
 				},
 				ExecutionManager::NativeWhenPossible => {
-					let res = self.execute_aux(compute_tx, true, native_call);
-					(res.0, res.2, res.3)
+					self.execute_aux(true, native_call).0
 				},
-			};
-			result.map(move |out| (out, storage_delta, changes_delta))
+			}
 		};
 
 		if result.is_ok() {
@@ -455,7 +447,8 @@ pub fn prove_execution<B, H, Exec>(
 ) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
 where
 	B: Backend<H>,
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor,
 {
 	let trie_backend = backend.as_trie_backend()
@@ -481,7 +474,8 @@ pub fn prove_execution_on_trie_backend<S, H, Exec>(
 ) -> Result<(Vec<u8>, StorageProof), Box<dyn Error>>
 where
 	S: trie_backend_essence::TrieBackendStorage<H>,
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor,
 {
 	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
@@ -489,9 +483,8 @@ where
 		&proving_backend, None, overlay, exec, method, call_data, Extensions::default(),
 	);
 
-	let (result, _, _) = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
+	let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 		always_wasm(),
-		false,
 		None,
 	)?;
 	let proof = sm.backend.extract_proof();
@@ -508,9 +501,9 @@ pub fn execution_proof_check<H, Exec>(
 	call_data: &[u8],
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
 	Exec: CodeExecutor,
-	H::Out: Ord + 'static,
+	H::Out: Ord + 'static + codec::Codec,
 {
 	let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
 	execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data)
@@ -525,7 +518,8 @@ pub fn execution_proof_check_on_trie_backend<H, Exec>(
 	call_data: &[u8],
 ) -> Result<Vec<u8>, Box<dyn Error>>
 where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor,
 {
 	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
@@ -534,9 +528,8 @@ where
 
 	sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 		always_untrusted_wasm(),
-		false,
 		None,
-	).map(|(result, _, _)| result.into_encoded())
+	).map(NativeOrEncoded::into_encoded)
 }
 
 /// Generate storage read proof.
@@ -546,8 +539,8 @@ pub fn prove_read<B, H, I>(
 ) -> Result<StorageProof, Box<dyn Error>>
 where
 	B: Backend<H>,
-	H: Hasher<Out=H256>,
-	H::Out: Ord,
+	H: Hasher,
+	H::Out: Ord + Codec,
 	I: IntoIterator,
 	I::Item: AsRef<[u8]>,
 {
@@ -739,7 +732,6 @@ mod tests {
 	use codec::Encode;
 	use overlayed_changes::OverlayedValue;
 	use super::*;
-	use super::backend::InMemory;
 	use super::ext::Ext;
 	use super::changes_trie::{
 		InMemoryStorage as InMemoryChangesTrieStorage,
@@ -825,7 +817,7 @@ mod tests {
 		);
 
 		assert_eq!(
-			state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap().0,
+			state_machine.execute(ExecutionStrategy::NativeWhenPossible).unwrap(),
 			vec![66],
 		);
 	}
@@ -852,7 +844,7 @@ mod tests {
 			Default::default(),
 		);
 
-		assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap().0, vec![66]);
+		assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap(), vec![66]);
 	}
 
 	#[test]
@@ -883,7 +875,6 @@ mod tests {
 					consensus_failed = true;
 					we
 				}),
-				true,
 				None,
 			).is_err()
 		);
@@ -933,7 +924,7 @@ mod tests {
 			b"abc".to_vec() => b"2".to_vec(),
 			b"bbb".to_vec() => b"3".to_vec()
 		];
-		let mut state = InMemory::<Blake2Hasher>::from(initial);
+		let mut state = InMemoryBackend::<Blake2Hasher>::from(initial);
 		let backend = state.as_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges {
 			committed: map![
@@ -949,8 +940,10 @@ mod tests {
 
 		{
 			let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
+			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
+				&mut cache,
 				backend,
 				Some(&changes_trie_storage),
 				None,
@@ -975,12 +968,14 @@ mod tests {
 
 	#[test]
 	fn set_child_storage_works() {
-		let mut state = InMemory::<Blake2Hasher>::default();
+		let mut state = InMemoryBackend::<Blake2Hasher>::default();
 		let backend = state.as_trie_backend().unwrap();
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 		let mut overlay = OverlayedChanges::default();
+		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
+			&mut cache,
 			backend,
 			Some(&changes_trie_storage),
 			None,
@@ -1104,8 +1099,10 @@ mod tests {
 		let mut transaction = {
 			let backend = test_trie();
 			let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
+			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
+				&mut cache,
 				&backend,
 				Some(&changes_trie_storage),
 				None,
@@ -1113,7 +1110,7 @@ mod tests {
 			ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec());
 			ext.set_child_storage(subtrie2, CHILD_INFO_2, b"abc".to_vec(), b"def".to_vec());
 			ext.storage_root();
-			(ext.transaction().0).0
+			cache.transaction.unwrap()
 		};
 		let mut duplicate = false;
 		for (k, (value, rc)) in transaction.drain().iter() {

substrate/primitives/state-machine/src/overlayed_changes.rs

@@ -16,14 +16,23 @@
 
 //! The overlayed changes to state.
 
+use crate::{
+	backend::Backend, ChangesTrieTransaction,
+	changes_trie::{
+		NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig, BlockNumber, build_changes_trie,
+		Storage as ChangesTrieStorage,
+	},
+};
+
 #[cfg(test)]
 use std::iter::FromIterator;
 use std::collections::{HashMap, BTreeMap, BTreeSet};
-use codec::Decode;
-use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
+use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo};
 use std::{mem, ops};
 
+use hash_db::Hasher;
+
 /// The overlayed changes to state to be queried on top of the backend.
 ///
 /// A transaction shares all prospective changes within an inner overlay
@@ -60,6 +69,92 @@ pub struct OverlayedChangeSet {
 	pub children: HashMap<Vec<u8>, (BTreeMap<Vec<u8>, OverlayedValue>, OwnedChildInfo)>,
 }
 
+/// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
+///
+/// This contains all the changes to the storage and transactions to apply theses changes to the
+/// backend.
+pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
+	/// All changes to the main storage.
+	///
+	/// A value of `None` means that it was deleted.
+	pub main_storage_changes: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	/// All changes to the child storages.
+	pub child_storage_changes: Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+	/// A transaction for the backend that contains all changes from
+	/// [`main_storage_changes`](Self::main_storage_changes) and from
+	/// [`child_storage_changes`](Self::child_storage_changes).
+	pub transaction: Transaction,
+	/// The storage root after applying the transaction.
+	pub transaction_storage_root: H::Out,
+	/// Contains the transaction for the backend for the changes trie.
+	///
+	/// If changes trie is disabled the value is set to `None`.
+	pub changes_trie_transaction: Option<ChangesTrieTransaction<H, N>>,
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
+	/// Deconstruct into the inner values
+	pub fn into_inner(self) -> (
+		Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+		Transaction,
+		H::Out,
+		Option<ChangesTrieTransaction<H, N>>,
+	) {
+		(
+			self.main_storage_changes,
+			self.child_storage_changes,
+			self.transaction,
+			self.transaction_storage_root,
+			self.changes_trie_transaction,
+		)
+	}
+}
+
+/// The storage transaction are calculated as part of the `storage_root` and
+/// `changes_trie_storage_root`. These transactions can be reused for importing the block into the
+/// storage. So, we cache them to not require a recomputation of those transactions.
+pub struct StorageTransactionCache<Transaction, H: Hasher, N: BlockNumber> {
+	/// Contains the changes for the main and the child storages as one transaction.
+	pub(crate) transaction: Option<Transaction>,
+	/// The storage root after applying the transaction.
+	pub(crate) transaction_storage_root: Option<H::Out>,
+	/// Contains the changes trie transaction.
+	pub(crate) changes_trie_transaction: Option<Option<ChangesTrieTransaction<H, N>>>,
+	/// The storage root after applying the changes trie transaction.
+	pub(crate) changes_trie_transaction_storage_root: Option<Option<H::Out>>,
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> StorageTransactionCache<Transaction, H, N> {
+	/// Reset the cached transactions.
+	pub fn reset(&mut self) {
+		*self = Self::default();
+	}
+}
+
+impl<Transaction, H: Hasher, N: BlockNumber> Default for StorageTransactionCache<Transaction, H, N> {
+	fn default() -> Self {
+		Self {
+			transaction: None,
+			transaction_storage_root: None,
+			changes_trie_transaction: None,
+			changes_trie_transaction_storage_root: None,
+		}
+	}
+}
+
+impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges<Transaction, H, N> {
+	fn default() -> Self {
+		Self {
+			main_storage_changes: Default::default(),
+			child_storage_changes: Default::default(),
+			transaction: Default::default(),
+			transaction_storage_root: Default::default(),
+			changes_trie_transaction: None,
+		}
+	}
+}
+
 #[cfg(test)]
 impl FromIterator<(Vec<u8>, OverlayedValue)> for OverlayedChangeSet {
 	fn from_iter<T: IntoIterator<Item = (Vec<u8>, OverlayedValue)>>(iter: T) -> Self {
@@ -105,7 +200,7 @@ impl OverlayedChanges {
 		true
 	}
 
-	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
 	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
 	/// value has been set.
 	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
@@ -114,7 +209,7 @@ impl OverlayedChanges {
 			.map(|x| x.value.as_ref().map(AsRef::as_ref))
 	}
 
-	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred
 	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
 	/// value has been set.
 	pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option<Option<&[u8]>> {
@@ -237,7 +332,7 @@ impl OverlayedChanges {
 			}
 		}
 
-		// Then do the same with keys from commited changes.
+		// Then do the same with keys from committed changes.
 		// NOTE that we are making changes in the prospective change set.
 		for key in self.committed.top.keys() {
 			if key.starts_with(prefix) {
@@ -338,15 +433,61 @@ impl OverlayedChanges {
 		impl Iterator<Item=(Vec<u8>, (impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>, OwnedChildInfo))>,
 	){
 		assert!(self.prospective.is_empty());
-		(self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
+		(
+			self.committed.top.into_iter().map(|(k, v)| (k, v.value)),
 			self.committed.children.into_iter()
-				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))))
+				.map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci)))
+		)
+	}
+
+	/// Convert this instance with all changes into a [`StorageChanges`] instance.
+	pub fn into_storage_changes<
+		B: Backend<H>, H: Hasher, N: BlockNumber, T: ChangesTrieStorage<H, N>
+	>(
+		self,
+		backend: &B,
+		changes_trie_storage: Option<&T>,
+		parent_hash: H::Out,
+		mut cache: StorageTransactionCache<B::Transaction, H, N>,
+	) -> Result<StorageChanges<B::Transaction, H, N>, String> where H::Out: Ord + Encode + 'static {
+		// If the transaction does not exist, we generate it.
+		if cache.transaction.is_none() {
+			self.storage_root(backend, &mut cache);
+		}
+
+		let (transaction, transaction_storage_root) = cache.transaction.take()
+			.and_then(|t| cache.transaction_storage_root.take().map(|tr| (t, tr)))
+			.expect("Transaction was be generated as part of `storage_root`; qed");
+
+		// If the transaction does not exist, we generate it.
+		if cache.changes_trie_transaction.is_none() {
+			self.changes_trie_root(
+				backend,
+				changes_trie_storage,
+				parent_hash,
+				false,
+				&mut cache,
+			).map_err(|_| "Failed to generate changes trie transaction")?;
+		}
+
+		let changes_trie_transaction = cache.changes_trie_transaction
+			.take()
+			.expect("Changes trie transaction was generated by `changes_trie_root`; qed");
+
+		let (main_storage_changes, child_storage_changes) = self.into_committed();
+
+		Ok(StorageChanges {
+			main_storage_changes: main_storage_changes.collect(),
+			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
+			transaction,
+			transaction_storage_root,
+			changes_trie_transaction,
+		})
 	}
 
 	/// Inserts storage entry responsible for current extrinsic index.
 	#[cfg(test)]
 	pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
-		use codec::Encode;
 		self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
 			value: Some(extrinsic_index.encode()),
 			extrinsics: None,
@@ -356,7 +497,7 @@ impl OverlayedChanges {
 	/// Returns current extrinsic index to use in changes trie construction.
 	/// None is returned if it is not set or changes trie config is not set.
 	/// Persistent value (from the backend) can be ignored because runtime must
-	/// set this index before first and unset after last extrinsic is executied.
+	/// set this index before first and unset after last extrinsic is executed.
 	/// Changes that are made outside of extrinsics, are marked with
 	/// `NO_EXTRINSIC_INDEX` index.
 	fn extrinsic_index(&self) -> Option<u32> {
@@ -369,6 +510,75 @@ impl OverlayedChanges {
 		}
 	}
 
+	/// Generate the storage root using `backend` and all changes from `prospective` and `committed`.
+	///
+	/// Returns the storage root and caches storage transaction in the given `cache`.
+	pub fn storage_root<H: Hasher, N: BlockNumber, B: Backend<H>>(
+		&self,
+		backend: &B,
+		cache: &mut StorageTransactionCache<B::Transaction, H, N>,
+	) -> H::Out
+		where H::Out: Ord + Encode,
+	{
+		let child_storage_keys = self.prospective.children.keys()
+				.chain(self.committed.children.keys());
+		let child_delta_iter = child_storage_keys.map(|storage_key|
+			(
+				storage_key.clone(),
+				self.committed.children.get(storage_key)
+					.into_iter()
+					.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
+					.chain(
+						self.prospective.children.get(storage_key)
+							.into_iter()
+							.flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone())))
+					),
+				self.child_info(storage_key).cloned()
+					.expect("child info initialized in either committed or prospective"),
+			)
+		);
+
+		// compute and memoize
+		let delta = self.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
+			.chain(self.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
+
+		let (root, transaction) = backend.full_storage_root(delta, child_delta_iter);
+
+		cache.transaction = Some(transaction);
+		cache.transaction_storage_root = Some(root);
+
+		root
+	}
+
+	/// Generate the changes trie root.
+	///
+	/// Returns the changes trie root and caches the storage transaction into the given `cache`.
+	///
+	/// # Panics
+	///
+	/// Panics on storage error, when `panic_on_storage_error` is set.
+	pub fn changes_trie_root<H: Hasher, N: BlockNumber, B: Backend<H>, T: ChangesTrieStorage<H, N>>(
+		&self,
+		backend: &B,
+		changes_trie_storage: Option<&T>,
+		parent_hash: H::Out,
+		panic_on_storage_error: bool,
+		cache: &mut StorageTransactionCache<B::Transaction, H, N>,
+	) -> Result<Option<H::Out>, ()> where H::Out: Ord + Encode + 'static {
+		build_changes_trie::<_, T, H, N>(
+			backend,
+			changes_trie_storage,
+			self,
+			parent_hash,
+			panic_on_storage_error,
+		).map(|r| {
+			let root = r.as_ref().map(|r| r.1).clone();
+			cache.changes_trie_transaction = Some(r.map(|(db, _, cache)| (db, cache)));
+			cache.changes_trie_transaction_storage_root = Some(root);
+			root
+		})
+	}
+
 	/// Get child info for a storage key.
 	/// Take the latest value so prospective first.
 	pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> {
@@ -445,7 +655,7 @@ mod tests {
 	use sp_core::{
 		Blake2Hasher, traits::Externalities, storage::well_known_keys::EXTRINSIC_INDEX,
 	};
-	use crate::backend::InMemory;
+	use crate::InMemoryBackend;
 	use crate::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
 	use crate::ext::Ext;
 	use super::*;
@@ -494,7 +704,7 @@ mod tests {
 			(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
 			(b"doug".to_vec(), b"notadog".to_vec()),
 		].into_iter().collect();
-		let backend = InMemory::<Blake2Hasher>::from(initial);
+		let backend = InMemoryBackend::<Blake2Hasher>::from(initial);
 		let mut overlay = OverlayedChanges {
 			committed: vec![
 				(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
@@ -509,8 +719,10 @@ mod tests {
 		};
 
 		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
+		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
+			&mut cache,
 			&backend,
 			Some(&changes_trie_storage),
 			None,

substrate/primitives/state-machine/src/proving_backend.rs

@@ -22,8 +22,8 @@ use codec::{Decode, Encode, Codec};
 use log::debug;
 use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix};
 use sp_trie::{
-	MemoryDB, PrefixedMemoryDB, default_child_trie_root,
-	read_trie_value_with, read_child_trie_value_with, record_all_keys
+	MemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with,
+	record_all_keys
 };
 pub use sp_trie::Recorder;
 pub use sp_trie::trie_types::{Layout, TrieError};
@@ -136,7 +136,7 @@ impl<'a, S, H> ProvingBackendRecorder<'a, S, H>
 			&eph,
 			self.backend.root(),
 			key,
-			&mut *self.proof_recorder
+			&mut *self.proof_recorder,
 		).map_err(map_e)
 	}
 
@@ -238,7 +238,9 @@ impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> ProvingBackend<'a, S, H>
 	}
 }
 
-impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H> for ProofRecorderBackend<'a, S, H> {
+impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H>
+	for ProofRecorderBackend<'a, S, H>
+{
 	type Overlay = S::Overlay;
 
 	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
@@ -251,7 +253,9 @@ impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> TrieBackendStorage<H> fo
 	}
 }
 
-impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> std::fmt::Debug for ProvingBackend<'a, S, H> {
+impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> std::fmt::Debug
+	for ProvingBackend<'a, S, H>
+{
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		write!(f, "ProvingBackend")
 	}
@@ -265,7 +269,7 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
 {
 	type Error = String;
 	type Transaction = S::Overlay;
-	type TrieBackendStorage = PrefixedMemoryDB<H>;
+	type TrieBackendStorage = S;
 
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
 		self.0.storage(key)
@@ -391,11 +395,12 @@ where
 
 #[cfg(test)]
 mod tests {
-	use crate::backend::{InMemory};
+	use crate::InMemoryBackend;
 	use crate::trie_backend::tests::test_trie;
 	use super::*;
 	use sp_core::{Blake2Hasher, storage::ChildStorageKey};
 	use crate::proving_backend::create_proof_check_backend;
+	use sp_trie::PrefixedMemoryDB;
 
 	const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1");
 	const CHILD_INFO_2: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_2");
@@ -446,7 +451,7 @@ mod tests {
 	#[test]
 	fn proof_recorded_and_checked() {
 		let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::<Vec<_>>();
-		let in_memory = InMemory::<Blake2Hasher>::default();
+		let in_memory = InMemoryBackend::<Blake2Hasher>::default();
 		let mut in_memory = in_memory.update(vec![(None, contents)]);
 		let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
 		(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
@@ -478,7 +483,7 @@ mod tests {
 			(Some((own2.clone(), CHILD_INFO_2.to_owned())),
 				(10..15).map(|i| (vec![i], Some(vec![i]))).collect()),
 		];
-		let in_memory = InMemory::<Blake2Hasher>::default();
+		let in_memory = InMemoryBackend::<Blake2Hasher>::default();
 		let mut in_memory = in_memory.update(contents);
 		let in_memory_root = in_memory.full_storage_root::<_, Vec<_>, _>(
 			::std::iter::empty(),
@@ -533,5 +538,4 @@ mod tests {
 			vec![64]
 		);
 	}
-
 }

substrate/primitives/state-machine/src/testing.rs

@@ -19,36 +19,45 @@
 use std::any::{Any, TypeId};
 use hash_db::Hasher;
 use crate::{
-	backend::{InMemory, Backend}, OverlayedChanges,
+	backend::Backend, OverlayedChanges, StorageTransactionCache, ext::Ext, InMemoryBackend,
 	changes_trie::{
 		InMemoryStorage as ChangesTrieInMemoryStorage,
 		BlockNumber as ChangesTrieBlockNumber,
 	},
-	ext::Ext,
 };
 use sp_core::{
 	storage::{
 		well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
 		Storage,
 	},
-	hash::H256, Blake2Hasher,
+	Blake2Hasher,
 };
 use codec::Encode;
 use sp_externalities::{Extensions, Extension};
 
 /// Simple HashMap-based Externalities impl.
-pub struct TestExternalities<H: Hasher<Out=H256>=Blake2Hasher, N: ChangesTrieBlockNumber=u64> {
+pub struct TestExternalities<H: Hasher = Blake2Hasher, N: ChangesTrieBlockNumber = u64>
+where
+	H::Out: codec::Codec,
+{
 	overlay: OverlayedChanges,
-	backend: InMemory<H>,
+	storage_transaction_cache: StorageTransactionCache<
+		<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
+	>,
+	backend: InMemoryBackend<H>,
 	changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
 	extensions: Extensions,
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
+impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
+	where
+		H::Out: Ord + 'static + codec::Codec
+{
 	/// Get externalities implementation.
-	pub fn ext(&mut self) -> Ext<H, N, InMemory<H>, ChangesTrieInMemoryStorage<H, N>> {
+	pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>, ChangesTrieInMemoryStorage<H, N>> {
 		Ext::new(
 			&mut self.overlay,
+			&mut self.storage_transaction_cache,
 			&self.backend,
 			Some(&self.changes_trie_storage),
 			Some(&mut self.extensions),
@@ -81,6 +90,7 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
 			backend: storage.into(),
 			extensions: Default::default(),
+			storage_transaction_cache: Default::default(),
 		}
 	}
 
@@ -100,7 +110,7 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 	}
 
 	/// Return a new backend with all pending value.
-	pub fn commit_all(&self) -> InMemory<H> {
+	pub fn commit_all(&self) -> InMemoryBackend<H> {
 		let top: Vec<_> = self.overlay.committed.top.clone().into_iter()
 			.chain(self.overlay.prospective.top.clone().into_iter())
 			.map(|(k, v)| (k, v.value)).collect();
@@ -129,13 +139,18 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 	}
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N> {
-	fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+impl<H: Hasher, N: ChangesTrieBlockNumber> std::fmt::Debug for TestExternalities<H, N>
+	where H::Out: codec::Codec,
+{
+	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
 		write!(f, "overlay: {:?}\nbackend: {:?}", self.overlay, self.backend.pairs())
 	}
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N> {
+impl<H: Hasher, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N>
+	where
+		H::Out: Ord + 'static + codec::Codec
+{
 	/// This doesn't test if they are in the same state, only if they contains the
 	/// same data at this state
 	fn eq(&self, other: &TestExternalities<H, N>) -> bool {
@@ -143,18 +158,25 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> PartialEq for TestExternali
 	}
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N> {
+impl<H: Hasher, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N>
+	where
+		H::Out: Ord + 'static + codec::Codec,
+{
 	fn default() -> Self { Self::new(Default::default()) }
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N> {
+impl<H: Hasher, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N>
+	where
+		H::Out: Ord + 'static + codec::Codec,
+{
 	fn from(storage: Storage) -> Self {
 		Self::new(storage)
 	}
 }
 
 impl<H, N> sp_externalities::ExtensionStore for TestExternalities<H, N> where
-	H: Hasher<Out=H256>,
+	H: Hasher,
+	H::Out: codec::Codec,
 	N: ChangesTrieBlockNumber,
 {
 	fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> {

substrate/primitives/state-machine/src/trie_backend_essence.rs

@@ -237,7 +237,6 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		self.keys_values_with_prefix_inner(&self.root, prefix, |k, _v| f(k), None)
 	}
 
-
 	fn keys_values_with_prefix_inner<F: FnMut(&[u8], &[u8])>(
 		&self,
 		root: &H::Out,
@@ -285,7 +284,6 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 	pub fn for_key_values_with_prefix<F: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], f: F) {
 		self.keys_values_with_prefix_inner(&self.root, prefix, f, None)
 	}
-
 }
 
 pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
@@ -293,20 +291,16 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
 	overlay: &'a mut S::Overlay,
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: 'a + Hasher
-> hash_db::AsPlainDB<H::Out, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> hash_db::AsPlainDB<H::Out, DBValue>
 	for Ephemeral<'a, S, H>
 {
 	fn as_plain_db<'b>(&'b self) -> &'b (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
-	fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) { self }
+	fn as_plain_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::PlainDB<H::Out, DBValue> + 'b) {
+		self
+	}
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: 'a + Hasher
-> hash_db::AsHashDB<H, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> hash_db::AsHashDB<H, DBValue>
 	for Ephemeral<'a, S, H>
 {
 	fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB<H, DBValue> + 'b) { self }
@@ -322,10 +316,7 @@ impl<'a, S: TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
 	}
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: Hasher
-> hash_db::PlainDB<H::Out, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::PlainDB<H::Out, DBValue>
 	for Ephemeral<'a, S, H>
 {
 	fn get(&self, key: &H::Out) -> Option<DBValue> {
@@ -355,20 +346,14 @@ impl<'a,
 	}
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: Hasher
-> hash_db::PlainDBRef<H::Out, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::PlainDBRef<H::Out, DBValue>
 	for Ephemeral<'a, S, H>
 {
 	fn get(&self, key: &H::Out) -> Option<DBValue> { hash_db::PlainDB::get(self, key) }
 	fn contains(&self, key: &H::Out) -> bool { hash_db::PlainDB::contains(self, key) }
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: Hasher
-> hash_db::HashDB<H, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::HashDB<H, DBValue>
 	for Ephemeral<'a, S, H>
 {
 	fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> {
@@ -402,14 +387,16 @@ impl<'a,
 	}
 }
 
-impl<'a,
-	S: 'a + TrieBackendStorage<H>,
-	H: Hasher
-> hash_db::HashDBRef<H, DBValue>
+impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> hash_db::HashDBRef<H, DBValue>
 	for Ephemeral<'a, S, H>
 {
-	fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> { hash_db::HashDB::get(self, key, prefix) }
-	fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { hash_db::HashDB::contains(self, key, prefix) }
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Option<DBValue> {
+		hash_db::HashDB::get(self, key, prefix)
+	}
+
+	fn contains(&self, key: &H::Out, prefix: Prefix) -> bool {
+		hash_db::HashDB::contains(self, key, prefix)
+	}
 }
 
 /// Key-value pairs storage that is used by trie backend essence.

substrate/primitives/test-primitives/src/lib.rs

@@ -40,7 +40,6 @@ impl serde::Serialize for Extrinsic {
 	}
 }
 
-
 impl ExtrinsicT for Extrinsic {
 	type Call = Extrinsic;
 	type SignaturePayload = ();
@@ -77,11 +76,10 @@ pub type Block = sp_runtime::generic::Block<Header, Extrinsic>;
 /// A test block's header.
 pub type Header = sp_runtime::generic::Header<BlockNumber, BlakeTwo256>;
 
-
 /// Changes trie configuration (optionally) used in tests.
 pub fn changes_trie_config() -> sp_core::ChangesTrieConfiguration {
 	sp_core::ChangesTrieConfiguration {
 		digest_interval: 4,
 		digest_levels: 2,
 	}
-}
+}

substrate/primitives/version/src/lib.rs

@@ -24,8 +24,6 @@ use serde::{Serialize, Deserialize};
 use std::fmt;
 #[cfg(feature = "std")]
 use std::collections::HashSet;
-#[cfg(feature = "std")]
-use sp_runtime::traits::RuntimeApiInfo;
 
 use codec::Encode;
 #[cfg(feature = "std")]
@@ -42,7 +40,7 @@ pub type ApiId = [u8; 8];
 /// A vector of pairs of `ApiId` and a `u32` for version. For `"std"` builds, this
 /// is a `Cow`.
 #[cfg(feature = "std")]
-pub type ApisVec = ::std::borrow::Cow<'static, [(ApiId, u32)]>;
+pub type ApisVec = std::borrow::Cow<'static, [(ApiId, u32)]>;
 /// A vector of pairs of `ApiId` and a `u32` for version. For `"no-std"` builds, this
 /// is just a reference.
 #[cfg(not(feature = "std"))]
@@ -131,21 +129,14 @@ impl RuntimeVersion {
 		self.authoring_version == other.authoring_version
 	}
 
-	/// Check if this version supports a particular API.
-	pub fn has_api<A: RuntimeApiInfo + ?Sized>(&self) -> bool {
-		self.apis.iter().any(|(s, v)| {
-			s == &A::ID && *v == A::VERSION
-		})
-	}
-
-	/// Check if the given api is implemented and the version passes a predicate.
-	pub fn has_api_with<A: RuntimeApiInfo + ?Sized, P: Fn(u32) -> bool>(
+	/// Check if the given api with `api_id` is implemented and the version passes the given
+	/// `predicate`.
+	pub fn has_api_with<P: Fn(u32) -> bool>(
 		&self,
-		pred: P,
+		id: &ApiId,
+		predicate: P,
 	) -> bool {
-		self.apis.iter().any(|(s, v)| {
-			s == &A::ID && pred(*v)
-		})
+		self.apis.iter().any(|(s, v)| s == id && predicate(*v))
 	}
 }