Remove requirement on Hash = H256, make Proposer return StorageChanges and Proof (#3860)

* Extend `Proposer` to optionally generate a proof of the proposal

* Something

* Refactor sr-api to not depend on client anymore

* Fix benches

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Apply suggestions from code review

* Introduce new `into_storage_changes` function

* Switch to runtime api for `execute_block` and don't require `H256`
anywhere in the code

* Put the `StorageChanges` into the `Proposal`

* Move the runtime api error to its own trait

* Adds `StorageTransactionCache` to the runtime api

This requires that we add `type NodeBlock = ` to the
`impl_runtime_apis!` macro to work around some bugs in rustc :(

* Remove `type NodeBlock` and switch to a "better" hack

* Start using the transaction cache from the runtime api

* Make it compile

* Move `InMemory` to its own file

* Make all tests work again

* Return block, storage_changes and proof from Blockbuilder::bake()

* Make sure that we use/set `storage_changes` when possible

* Add test

* Fix deadlock

* Remove accidentally added folders

* Introduce `RecordProof` as argument type to be more explicit

* Update client/src/client.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update primitives/state-machine/src/ext.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Integrates review feedback

* Remove `unsafe` usage

* Update client/block-builder/src/lib.rs

Co-Authored-By: Benjamin Kampmann <ben@gnunicorn.org>

* Update client/src/call_executor.rs

* Bump versions

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
Co-authored-by: Benjamin Kampmann <ben.kampmann@googlemail.com>

substrate/primitives/api/Cargo.toml

@@ -12,6 +12,7 @@ sp-std = { version = "2.0.0", default-features = false, path = "../std" }
 sp-runtime = { version = "2.0.0", default-features = false, path = "../runtime" }
 sp-version = { version = "2.0.0", default-features = false, path = "../version" }
 sp-state-machine = { version = "2.0.0", optional = true, path = "../../primitives/state-machine" }
+hash-db = { version = "0.15.2", optional = true }
 
 [dev-dependencies]
 sp-test-primitives = { version = "2.0.0", path = "../test-primitives" }
@@ -25,4 +26,5 @@ std = [
 	"sp-runtime/std",
 	"sp-state-machine",
 	"sp-version/std",
+	"hash-db",
 ]

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -16,7 +16,7 @@
 
 use crate::utils::{
 	generate_crate_access, generate_hidden_includes, generate_runtime_mod_name_for_trait,
-	fold_fn_decl_for_client_side, unwrap_or_error, extract_parameter_names_types_and_borrows,
+	fold_fn_decl_for_client_side, extract_parameter_names_types_and_borrows,
 	generate_native_call_generator_fn_name, return_type_extract_type,
 	generate_method_runtime_api_impl_name, generate_call_api_at_fn_name, prefix_function_with_trait,
 	replace_wild_card_parameter_names,
@@ -399,7 +399,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				R: #crate_::Encode + #crate_::Decode + PartialEq,
 				NC: FnOnce() -> std::result::Result<R, String> + std::panic::UnwindSafe,
 				Block: #crate_::BlockT,
-				T: #crate_::CallRuntimeAt<Block>,
+				T: #crate_::CallApiAt<Block>,
 				C: #crate_::Core<Block, Error = T::Error>,
 			>(
 				call_runtime_at: &T,
@@ -407,6 +407,9 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
+				storage_transaction_cache: &std::cell::RefCell<
+					#crate_::StorageTransactionCache<Block, T::StateBackend>
+				>,
 				initialized_block: &std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 				native_call: Option<NC>,
 				context: #crate_::ExecutionContext,
@@ -426,34 +429,40 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 					if version.apis.iter().any(|(s, v)| {
 						s == &ID && *v < #versions
 					}) {
-						let ret = call_runtime_at.call_api_at::<R, fn() -> _, _>(
+						let params = #crate_::CallApiAtParams::<_, _, fn() -> _, _> {
 							core_api,
 							at,
-							#old_names,
-							args,
-							changes,
+							function: #old_names,
+							native_call: None,
+							arguments: args,
+							overlayed_changes: changes,
+							storage_transaction_cache,
 							initialize_block,
-							None,
 							context,
 							recorder,
-						)?;
+						};
+
+						let ret = call_runtime_at.call_api_at(params)?;
 
 						update_initialized_block();
-						return Ok(ret);
+						return Ok(ret)
 					}
 				)*
 
-				let ret = call_runtime_at.call_api_at(
+				let params = #crate_::CallApiAtParams {
 					core_api,
 					at,
-					#trait_fn_name,
-					args,
-					changes,
-					initialize_block,
+					function: #trait_fn_name,
 					native_call,
+					arguments: args,
+					overlayed_changes: changes,
+					storage_transaction_cache,
+					initialize_block,
 					context,
 					recorder,
-				)?;
+				};
+
+				let ret = call_runtime_at.call_api_at(params)?;
 
 				update_initialized_block();
 				Ok(ret)
@@ -465,7 +474,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 }
 
 /// Generate the declaration of the trait for the runtime.
-fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
+fn generate_runtime_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 	let mut result = Vec::new();
 
 	for decl in decls {
@@ -473,12 +482,12 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 		extend_generics_with_block(&mut decl.generics);
 		let mod_name = generate_runtime_mod_name_for_trait(&decl.ident);
 		let found_attributes = remove_supported_attributes(&mut decl.attrs);
-		let api_version = unwrap_or_error(get_api_version(&found_attributes).map(|v| {
+		let api_version = get_api_version(&found_attributes).map(|v| {
 			generate_runtime_api_version(v as u32)
-		}));
+		})?;
 		let id = generate_runtime_api_id(&decl.ident.to_string());
 
-		let call_api_at_calls = unwrap_or_error(generate_call_api_at_calls(&decl));
+		let call_api_at_calls = generate_call_api_at_calls(&decl)?;
 
 		// Remove methods that have the `changed_in` attribute as they are not required for the
 		// runtime anymore.
@@ -495,7 +504,7 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 			r => Some(r.clone()),
 		}).collect();
 
-		let native_call_generators = unwrap_or_error(generate_native_call_generators(&decl));
+		let native_call_generators = generate_native_call_generators(&decl)?;
 
 		result.push(quote!(
 			#[doc(hidden)]
@@ -517,7 +526,7 @@ fn generate_runtime_decls(decls: &[ItemTrait]) -> TokenStream {
 		));
 	}
 
-	quote!( #( #result )* )
+	Ok(quote!( #( #result )* ))
 }
 
 /// Modify the given runtime api declaration to be usable on the client side.
@@ -722,7 +731,7 @@ impl<'a> Fold for ToClientSideDecl<'a> {
 				'static
 				+ Send
 				+ Sync
-				+ #crate_::ApiExt<#block_ident>
+				+ #crate_::ApiErrorExt
 			);
 		} else {
 			// Add the `Core` runtime api as super trait.
@@ -823,7 +832,7 @@ fn get_api_version(found_attributes: &HashMap<&'static str, Attribute>) -> Resul
 }
 
 /// Generate the declaration of the trait for the client side.
-fn generate_client_side_decls(decls: &[ItemTrait]) -> TokenStream {
+fn generate_client_side_decls(decls: &[ItemTrait]) -> Result<TokenStream> {
 	let mut result = Vec::new();
 
 	for decl in decls {
@@ -848,14 +857,12 @@ fn generate_client_side_decls(decls: &[ItemTrait]) -> TokenStream {
 
 		let api_version = get_api_version(&found_attributes);
 
-		let runtime_info = unwrap_or_error(
-			api_version.map(|v| generate_runtime_info_impl(&decl, v))
-		);
+		let runtime_info = api_version.map(|v| generate_runtime_info_impl(&decl, v))?;
 
 		result.push(quote!( #decl #runtime_info #( #errors )* ));
 	}
 
-	quote!( #( #result )* )
+	Ok(quote!( #( #result )* ))
 }
 
 /// Checks that a trait declaration is in the format we expect.
@@ -908,15 +915,17 @@ impl<'ast> Visit<'ast> for CheckTraitDecl {
 }
 
 /// Check that the trait declarations are in the format we expect.
-fn check_trait_decls(decls: &[ItemTrait]) -> Option<TokenStream> {
+fn check_trait_decls(decls: &[ItemTrait]) -> Result<()> {
 	let mut checker = CheckTraitDecl { errors: Vec::new() };
 	decls.iter().for_each(|decl| visit::visit_item_trait(&mut checker, &decl));
 
-	if checker.errors.is_empty() {
-		None
+	if let Some(err) = checker.errors.pop() {
+		Err(checker.errors.into_iter().fold(err, |mut err, other| {
+			err.combine(other);
+			err
+		}))
 	} else {
-		let errors = checker.errors.into_iter().map(|e| e.to_compile_error());
-		Some(quote!( #( #errors )* ))
+		Ok(())
 	}
 }
 
@@ -925,19 +934,23 @@ pub fn decl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::Tok
 	// Parse all trait declarations
 	let RuntimeApiDecls { decls: api_decls } = parse_macro_input!(input as RuntimeApiDecls);
 
-	if let Some(errors) = check_trait_decls(&api_decls) {
-		return errors.into();
-	}
+	decl_runtime_apis_impl_inner(&api_decls).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+fn decl_runtime_apis_impl_inner(api_decls: &[ItemTrait]) -> Result<TokenStream> {
+	check_trait_decls(&api_decls)?;
 
 	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
-	let runtime_decls = generate_runtime_decls(&api_decls);
-	let client_side_decls = generate_client_side_decls(&api_decls);
+	let runtime_decls = generate_runtime_decls(api_decls)?;
+	let client_side_decls = generate_client_side_decls(api_decls)?;
 
-	quote!(
-		#hidden_includes
+	Ok(
+		quote!(
+			#hidden_includes
 
-		#runtime_decls
+			#runtime_decls
 
-		#client_side_decls
-	).into()
+			#client_side_decls
+		)
+	)
 }

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -15,10 +15,11 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::utils::{
-	unwrap_or_error, generate_crate_access, generate_hidden_includes,
+	generate_crate_access, generate_hidden_includes,
 	generate_runtime_mod_name_for_trait, generate_method_runtime_api_impl_name,
 	extract_parameter_names_types_and_borrows, generate_native_call_generator_fn_name,
 	return_type_extract_type, generate_call_api_at_fn_name, prefix_function_with_trait,
+	extract_all_signature_types,
 };
 
 use proc_macro2::{Span, TokenStream};
@@ -28,7 +29,7 @@ use quote::quote;
 use syn::{
 	spanned::Spanned, parse_macro_input, Ident, Type, ItemImpl, Path, Signature,
 	ImplItem, parse::{Parse, ParseStream, Result, Error}, PathArguments, GenericArgument, TypePath,
-	fold::{self, Fold}, parse_quote
+	fold::{self, Fold}, parse_quote,
 };
 
 use std::{collections::HashSet, iter};
@@ -49,7 +50,11 @@ impl Parse for RuntimeApiImpls {
 			impls.push(ItemImpl::parse(input)?);
 		}
 
-		Ok(Self { impls })
+		if impls.is_empty() {
+			Err(Error::new(Span::call_site(), "No api implementation given!"))
+		} else {
+			Ok(Self { impls })
+		}
 	}
 }
 
@@ -222,59 +227,69 @@ fn generate_wasm_interface(impls: &[ItemImpl]) -> Result<TokenStream> {
 	Ok(quote!( #( #impl_calls )* ))
 }
 
-fn generate_block_and_block_id_ty(
-	runtime: &Type,
-	trait_: &'static str,
-	assoc_type: &'static str,
-) -> (TokenStream, TokenStream) {
+fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-	let trait_ = Ident::new(trait_, Span::call_site());
-	let assoc_type = Ident::new(assoc_type, Span::call_site());
-
-	let block = quote!( <#runtime as #crate_::#trait_>::#assoc_type );
-	let block_id = quote!( #crate_::BlockId<#block> );
-
-	(block, block_id)
-}
-
-fn generate_node_block_and_block_id_ty(runtime: &Type) -> (TokenStream, TokenStream) {
-	generate_block_and_block_id_ty(runtime, "GetNodeBlockType", "NodeBlock")
-}
-
-fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStream> {
-	let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-	let runtime = &impls.get(0).ok_or_else(||
-		Error::new(Span::call_site(), "No api implementation given!")
-	)?.self_ty;
-	let (block, block_id) = generate_node_block_and_block_id_ty(runtime);
 
 	Ok(quote!(
 		pub struct RuntimeApi {}
 		/// Implements all runtime apis for the client side.
 		#[cfg(any(feature = "std", test))]
-		pub struct RuntimeApiImpl<C: #crate_::CallRuntimeAt<#block> + 'static> {
+		pub struct RuntimeApiImpl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block> + 'static>
+			where
+				// Rust bug: https://github.com/rust-lang/rust/issues/24159
+				C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			call: &'static C,
 			commit_on_success: std::cell::RefCell<bool>,
-			initialized_block: std::cell::RefCell<Option<#block_id>>,
+			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			recorder: Option<#crate_::ProofRecorder<#block>>,
+			storage_transaction_cache: std::cell::RefCell<
+				#crate_::StorageTransactionCache<Block, C::StateBackend>
+			>,
+			recorder: Option<#crate_::ProofRecorder<Block>>,
 		}
 
 		// `RuntimeApi` itself is not threadsafe. However, an instance is only available in a
 		// `ApiRef` object and `ApiRef` also has an associated lifetime. This lifetimes makes it
 		// impossible to move `RuntimeApi` into another thread.
 		#[cfg(any(feature = "std", test))]
-		unsafe impl<C: #crate_::CallRuntimeAt<#block>> Send for RuntimeApiImpl<C> {}
-		#[cfg(any(feature = "std", test))]
-		unsafe impl<C: #crate_::CallRuntimeAt<#block>> Sync for RuntimeApiImpl<C> {}
+		unsafe impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> Send
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block>> #crate_::ApiExt<#block> for RuntimeApiImpl<C> {
+		unsafe impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> Sync
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{}
+
+		#[cfg(any(feature = "std", test))]
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> #crate_::ApiErrorExt
+			for RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			type Error = C::Error;
+		}
+
+		#[cfg(any(feature = "std", test))]
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> #crate_::ApiExt<Block> for
+			RuntimeApiImpl<Block, C>
+				where
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
+			type StateBackend = C::StateBackend;
 
 			fn map_api_result<F: FnOnce(&Self) -> std::result::Result<R, E>, R, E>(
 				&self,
-				map_call: F
+				map_call: F,
 			) -> std::result::Result<R, E> where Self: Sized {
 				*self.commit_on_success.borrow_mut() = false;
 				let res = map_call(self);
@@ -287,7 +302,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 
 			fn runtime_version_at(
 				&self,
-				at: &#block_id
+				at: &#crate_::BlockId<Block>,
 			) -> std::result::Result<#crate_::RuntimeVersion, C::Error> {
 				self.call.runtime_version_at(at)
 			}
@@ -307,13 +322,37 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 						#crate_::StorageProof::new(trie_nodes)
 					})
 			}
+
+			fn into_storage_changes<
+				T: #crate_::ChangesTrieStorage<#crate_::HasherFor<Block>, #crate_::NumberFor<Block>>
+			>(
+				&self,
+				backend: &Self::StateBackend,
+				changes_trie_storage: Option<&T>,
+				parent_hash: Block::Hash,
+			) -> std::result::Result<
+				#crate_::StorageChanges<Self::StateBackend, Block>,
+				String
+			> where Self: Sized {
+				self.initialized_block.borrow_mut().take();
+				self.changes.replace(Default::default()).into_storage_changes(
+					backend,
+					changes_trie_storage,
+					parent_hash,
+					self.storage_transaction_cache.replace(Default::default()),
+				)
+			}
 		}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block> + 'static> #crate_::ConstructRuntimeApi<#block, C>
+		impl<Block: #crate_::BlockT, C> #crate_::ConstructRuntimeApi<Block, C>
 			for RuntimeApi
+				where
+					C: #crate_::CallApiAt<Block> + 'static,
+					// Rust bug: https://github.com/rust-lang/rust/issues/24159
+					C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
 		{
-			type RuntimeApi = RuntimeApiImpl<C>;
+			type RuntimeApi = RuntimeApiImpl<Block, C>;
 
 			fn construct_runtime_api<'a>(
 				call: &'a C,
@@ -324,20 +363,26 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					initialized_block: None.into(),
 					changes: Default::default(),
 					recorder: Default::default(),
+					storage_transaction_cache: Default::default(),
 				}.into()
 			}
 		}
 
 		#[cfg(any(feature = "std", test))]
-		impl<C: #crate_::CallRuntimeAt<#block>> RuntimeApiImpl<C> {
+		impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> RuntimeApiImpl<Block, C>
+			where
+				// Rust bug: https://github.com/rust-lang/rust/issues/24159
+				C::StateBackend: #crate_::StateBackend<#crate_::HasherFor<Block>>,
+		{
 			fn call_api_at<
 				R: #crate_::Encode + #crate_::Decode + PartialEq,
 				F: FnOnce(
 					&C,
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
-					&std::cell::RefCell<Option<#crate_::BlockId<#block>>>,
-					&Option<#crate_::ProofRecorder<#block>>,
+					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
+					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
+					&Option<#crate_::ProofRecorder<Block>>,
 				) -> std::result::Result<#crate_::NativeOrEncoded<R>, E>,
 				E,
 			>(
@@ -348,6 +393,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					&self.call,
 					self,
 					&self.changes,
+					&self.storage_transaction_cache,
 					&self.initialized_block,
 					&self.recorder,
 				);
@@ -413,9 +459,7 @@ fn generate_api_impl_for_runtime(impls: &[ItemImpl]) -> Result<TokenStream> {
 /// `impl Api for Runtime` with `impl Api for RuntimeApi` and replace the method implementations
 /// with code that calls into the runtime.
 struct ApiRuntimeImplToApiRuntimeApiImpl<'a> {
-	node_block: &'a TokenStream,
 	runtime_block: &'a TypePath,
-	node_block_id: &'a TokenStream,
 	runtime_mod_path: &'a Path,
 	runtime_type: &'a Type,
 	trait_generic_arguments: &'a [GenericArgument],
@@ -425,8 +469,7 @@ struct ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	fn fold_type_path(&mut self, input: TypePath) -> TypePath {
 		let new_ty_path = if input == *self.runtime_block {
-			let node_block = self.node_block;
-			parse_quote!( #node_block )
+			parse_quote!( __SR_API_BLOCK__ )
 		} else {
 			input
 		};
@@ -442,9 +485,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 				generate_native_call_generator_fn_name(&input.sig.ident);
 			let call_api_at_call = generate_call_api_at_fn_name(&input.sig.ident);
 			let trait_generic_arguments = self.trait_generic_arguments;
-			let node_block = self.node_block;
 			let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-			let block_id = self.node_block_id;
 
 			// Generate the access to the native parameters
 			let param_tuple_access = if input.sig.inputs.len() == 1 {
@@ -471,7 +512,7 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 			// Rewrite the input parameters.
 			input.sig.inputs = parse_quote! {
 				&self,
-				at: &#block_id,
+				at: &#crate_::BlockId<__SR_API_BLOCK__>,
 				context: #crate_::ExecutionContext,
 				params: Option<( #( #param_types ),* )>,
 				params_encoded: Vec<u8>,
@@ -495,17 +536,25 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 					#error
 
 					self.call_api_at(
-						|call_runtime_at, core_api, changes, initialized_block, recorder| {
+						|
+							call_runtime_at,
+							core_api,
+							changes,
+							storage_transaction_cache,
+							initialized_block,
+							recorder
+						| {
 							#runtime_mod_path #call_api_at_call(
 								call_runtime_at,
 								core_api,
 								at,
 								params_encoded,
 								changes,
+								storage_transaction_cache,
 								initialized_block,
 								params.map(|p| {
 									#runtime_mod_path #native_call_generator_ident ::
-										<#runtime, #node_block #(, #trait_generic_arguments )*> (
+										<#runtime, __SR_API_BLOCK__ #(, #trait_generic_arguments )*> (
 										#( #param_tuple_access ),*
 									)
 								}),
@@ -526,13 +575,50 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 	}
 
 	fn fold_item_impl(&mut self, mut input: ItemImpl) -> ItemImpl {
-		// Implement the trait for the `RuntimeApiImpl`
-		input.self_ty = Box::new(parse_quote!( RuntimeApiImpl<RuntimeApiImplCall> ));
+		// All this `UnwindSafe` magic below here is required for this rust bug:
+		// https://github.com/rust-lang/rust/issues/24159
+		// Before we directly had the final block type and rust could determine that it is unwind
+		// safe, but now we just have a generic parameter `Block`.
 
 		let crate_ = generate_crate_access(HIDDEN_INCLUDES_ID);
-		let block = self.node_block;
+
+		// Implement the trait for the `RuntimeApiImpl`
+		input.self_ty = Box::new(
+			parse_quote!( RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall> )
+		);
+
 		input.generics.params.push(
-			parse_quote!( RuntimeApiImplCall: #crate_::CallRuntimeAt<#block> + 'static )
+			parse_quote!(
+				__SR_API_BLOCK__: #crate_::BlockT + std::panic::UnwindSafe +
+					std::panic::RefUnwindSafe
+			)
+		);
+		input.generics.params.push(
+			parse_quote!( RuntimeApiImplCall: #crate_::CallApiAt<__SR_API_BLOCK__> + 'static )
+		);
+
+		let where_clause = input.generics.make_where_clause();
+
+		where_clause.predicates.push(
+			parse_quote! {
+				RuntimeApiImplCall::StateBackend:
+					#crate_::StateBackend<#crate_::HasherFor<__SR_API_BLOCK__>>
+			}
+		);
+
+		// Require that all types used in the function signatures are unwind safe.
+		extract_all_signature_types(&input.items).iter().for_each(|i| {
+			where_clause.predicates.push(
+				parse_quote! {
+					#i: std::panic::UnwindSafe + std::panic::RefUnwindSafe
+				}
+			);
+		});
+
+		where_clause.predicates.push(
+			parse_quote! {
+				__SR_API_BLOCK__::Header: std::panic::UnwindSafe + std::panic::RefUnwindSafe
+			}
 		);
 
 		// The implementation for the `RuntimeApiImpl` is only required when compiling with
@@ -555,7 +641,6 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result<TokenStream>
 			.ok_or_else(|| Error::new(impl_trait_path.span(), "Empty trait path not possible!"))?
 			.clone();
 		let runtime_block = extract_runtime_block_ident(impl_trait_path)?;
-		let (node_block, node_block_id) = generate_node_block_and_block_id_ty(&impl_.self_ty);
 		let runtime_type = &impl_.self_ty;
 		let mut runtime_mod_path = extend_with_runtime_decl_path(impl_trait_path.clone());
 		// remove the trait to get just the module path
@@ -568,8 +653,6 @@ fn generate_api_impl_for_runtime_api(impls: &[ItemImpl]) -> Result<TokenStream>
 
 		let mut visitor = ApiRuntimeImplToApiRuntimeApiImpl {
 			runtime_block,
-			node_block: &node_block,
-			node_block_id: &node_block_id,
 			runtime_mod_path: &runtime_mod_path,
 			runtime_type: &*runtime_type,
 			trait_generic_arguments: &trait_generic_arguments,
@@ -627,31 +710,37 @@ pub fn impl_runtime_apis_impl(input: proc_macro::TokenStream) -> proc_macro::Tok
 	// Parse all impl blocks
 	let RuntimeApiImpls { impls: api_impls } = parse_macro_input!(input as RuntimeApiImpls);
 
-	let dispatch_impl = unwrap_or_error(generate_dispatch_function(&api_impls));
-	let api_impls_for_runtime = unwrap_or_error(generate_api_impl_for_runtime(&api_impls));
-	let base_runtime_api = unwrap_or_error(generate_runtime_api_base_structures(&api_impls));
-	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
-	let runtime_api_versions = unwrap_or_error(generate_runtime_api_versions(&api_impls));
-	let wasm_interface = unwrap_or_error(generate_wasm_interface(&api_impls));
-	let api_impls_for_runtime_api = unwrap_or_error(generate_api_impl_for_runtime_api(&api_impls));
-
-	quote!(
-		#hidden_includes
-
-		#base_runtime_api
-
-		#api_impls_for_runtime
-
-		#api_impls_for_runtime_api
-
-		#runtime_api_versions
-
-		pub mod api {
-			use super::*;
-
-			#dispatch_impl
-
-			#wasm_interface
-		}
-	).into()
+	impl_runtime_apis_impl_inner(&api_impls).unwrap_or_else(|e| e.to_compile_error()).into()
+}
+
+fn impl_runtime_apis_impl_inner(api_impls: &[ItemImpl]) -> Result<TokenStream> {
+	let dispatch_impl = generate_dispatch_function(api_impls)?;
+	let api_impls_for_runtime = generate_api_impl_for_runtime(api_impls)?;
+	let base_runtime_api = generate_runtime_api_base_structures()?;
+	let hidden_includes = generate_hidden_includes(HIDDEN_INCLUDES_ID);
+	let runtime_api_versions = generate_runtime_api_versions(api_impls)?;
+	let wasm_interface = generate_wasm_interface(api_impls)?;
+	let api_impls_for_runtime_api = generate_api_impl_for_runtime_api(api_impls)?;
+
+	Ok(
+		quote!(
+			#hidden_includes
+
+			#base_runtime_api
+
+			#api_impls_for_runtime
+
+			#api_impls_for_runtime_api
+
+			#runtime_api_versions
+
+			pub mod api {
+				use super::*;
+
+				#dispatch_impl
+
+				#wasm_interface
+			}
+		)
+	)
 }

substrate/primitives/api/proc-macro/src/utils.rs

@@ -18,6 +18,7 @@ use proc_macro2::{TokenStream, Span};
 
 use syn::{
 	Result, Ident, Signature, parse_quote, Type, Pat, spanned::Spanned, FnArg, Error, token::And,
+	ImplItem, ReturnType,
 };
 
 use quote::quote;
@@ -26,11 +27,6 @@ use std::env;
 
 use proc_macro_crate::crate_name;
 
-/// Unwrap the given result, if it is an error, `compile_error!` will be generated.
-pub fn unwrap_or_error(res: Result<TokenStream>) -> TokenStream {
-	res.unwrap_or_else(|e| e.to_compile_error())
-}
-
 fn generate_hidden_includes_mod_name(unique_id: &'static str) -> Ident {
 	Ident::new(&format!("sp_api_hidden_includes_{}", unique_id), Span::call_site())
 }
@@ -81,10 +77,10 @@ pub fn generate_method_runtime_api_impl_name(trait_: &Ident, method: &Ident) ->
 }
 
 /// Get the type of a `syn::ReturnType`.
-pub fn return_type_extract_type(rt: &syn::ReturnType) -> Type {
+pub fn return_type_extract_type(rt: &ReturnType) -> Type {
 	match rt {
-		syn::ReturnType::Default => parse_quote!( () ),
-		syn::ReturnType::Type(_, ref ty) => *ty.clone(),
+		ReturnType::Default => parse_quote!( () ),
+		ReturnType::Type(_, ref ty) => *ty.clone(),
 	}
 }
 
@@ -176,3 +172,30 @@ pub fn generate_call_api_at_fn_name(fn_name: &Ident) -> Ident {
 pub fn prefix_function_with_trait<F: ToString>(trait_: &Ident, function: &F) -> String {
 	format!("{}_{}", trait_.to_string(), function.to_string())
 }
+
+/// Extract all types that appear in signatures in the given `ImplItem`'s.
+///
+/// If a type is a reference, the inner type is extracted (without the reference).
+pub fn extract_all_signature_types(items: &[ImplItem]) -> Vec<Type> {
+	items.iter()
+		.filter_map(|i| match i {
+			ImplItem::Method(method) => Some(&method.sig),
+			_ => None,
+		})
+		.map(|sig| {
+			let ret_ty = match &sig.output {
+				ReturnType::Default => None,
+				ReturnType::Type(_, ty) => Some((**ty).clone()),
+			};
+
+			sig.inputs.iter().filter_map(|i| match i {
+				FnArg::Typed(arg) => Some(&arg.ty),
+				_ => None,
+			}).map(|ty| match &**ty {
+				Type::Reference(t) => (*t.elem).clone(),
+				_ => (**ty).clone(),
+			}).chain(ret_ty)
+		})
+		.flatten()
+		.collect()
+}

substrate/primitives/api/src/lib.rs

@@ -26,7 +26,7 @@
 //! functionality that every runtime needs to export.
 //!
 //! Besides the macros and the [`Core`] runtime api, this crates provides the [`Metadata`] runtime
-//! api, the [`ApiExt`] trait, the [`CallRuntimeAt`] trait and the [`ConstructRuntimeApi`] trait.
+//! api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi`] trait.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
@@ -35,18 +35,23 @@ extern crate self as sp_api;
 
 #[doc(hidden)]
 #[cfg(feature = "std")]
-pub use sp_state_machine::{OverlayedChanges, StorageProof};
+pub use sp_state_machine::{
+	OverlayedChanges, StorageProof, Backend as StateBackend, ChangesTrieStorage,
+};
 #[doc(hidden)]
 #[cfg(feature = "std")]
 pub use sp_core::NativeOrEncoded;
 #[doc(hidden)]
+#[cfg(feature = "std")]
+pub use hash_db::Hasher;
+#[doc(hidden)]
 #[cfg(not(feature = "std"))]
 pub use sp_core::to_substrate_wasm_fn_return_value;
 #[doc(hidden)]
 pub use sp_runtime::{
 	traits::{
-		Block as BlockT, GetNodeBlockType, GetRuntimeBlockType,
-		Header as HeaderT, ApiRef, RuntimeApiInfo, Hash as HashT,
+		Block as BlockT, GetNodeBlockType, GetRuntimeBlockType, HasherFor, NumberFor,
+		Header as HeaderT, Hash as HashT,
 	},
 	generic::BlockId, transaction_validity::TransactionValidity,
 };
@@ -157,8 +162,8 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// ```rust
 /// use sp_version::create_runtime_str;
 /// #
-/// # use sp_runtime::traits::GetNodeBlockType;
-/// # use sp_test_primitives::{Block, Header};
+/// # use sp_runtime::traits::{GetNodeBlockType, Block as BlockT};
+/// # use sp_test_primitives::Block;
 /// #
 /// # /// The declaration of the `Runtime` type and the implementation of the `GetNodeBlockType`
 /// # /// trait are done by the `construct_runtime!` macro in a real runtime.
@@ -187,7 +192,7 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// #           unimplemented!()
 /// #       }
 /// #       fn execute_block(_block: Block) {}
-/// #       fn initialize_block(_header: &Header) {}
+/// #       fn initialize_block(_header: &<Block as BlockT>::Header) {}
 /// #   }
 ///
 ///     impl self::Balance<Block> for Runtime {
@@ -221,27 +226,60 @@ pub use sp_api_proc_macro::decl_runtime_apis;
 /// ```
 pub use sp_api_proc_macro::impl_runtime_apis;
 
-#[cfg(feature = "std")]
 /// A type that records all accessed trie nodes and generates a proof out of it.
+#[cfg(feature = "std")]
 pub type ProofRecorder<B> = sp_state_machine::ProofRecorder<
 	<<<B as BlockT>::Header as HeaderT>::Hashing as HashT>::Hasher
 >;
 
+/// A type that is used as cache for the storage transactions.
+#[cfg(feature = "std")]
+pub type StorageTransactionCache<Block, Backend> =
+	sp_state_machine::StorageTransactionCache<
+		<Backend as StateBackend<HasherFor<Block>>>::Transaction, HasherFor<Block>, NumberFor<Block>
+	>;
+
+#[cfg(feature = "std")]
+pub type StorageChanges<SBackend, Block> =
+	sp_state_machine::StorageChanges<
+		<SBackend as StateBackend<HasherFor<Block>>>::Transaction,
+		HasherFor<Block>,
+		NumberFor<Block>
+	>;
+
+/// Extract the state backend type for a type that implements `ProvideRuntimeApi`.
+#[cfg(feature = "std")]
+pub type StateBackendFor<P, Block> =
+	<<P as ProvideRuntimeApi<Block>>::Api as ApiExt<Block>>::StateBackend;
+
+/// Extract the state backend transaction type for a type that implements `ProvideRuntimeApi`.
+#[cfg(feature = "std")]
+pub type TransactionFor<P, Block> =
+	<StateBackendFor<P, Block> as StateBackend<HasherFor<Block>>>::Transaction;
+
 /// Something that can be constructed to a runtime api.
 #[cfg(feature = "std")]
-pub trait ConstructRuntimeApi<Block: BlockT, C: CallRuntimeAt<Block>> {
+pub trait ConstructRuntimeApi<Block: BlockT, C: CallApiAt<Block>> {
 	/// The actual runtime api that will be constructed.
-	type RuntimeApi;
+	type RuntimeApi: ApiExt<Block>;
 
 	/// Construct an instance of the runtime api.
 	fn construct_runtime_api<'a>(call: &'a C) -> ApiRef<'a, Self::RuntimeApi>;
 }
 
-/// An extension for the `RuntimeApi`.
+/// Extends the runtime api traits with an associated error type. This trait is given as super
+/// trait to every runtime api trait.
 #[cfg(feature = "std")]
-pub trait ApiExt<Block: BlockT> {
-	/// Error type used by the interface.
+pub trait ApiErrorExt {
+	/// Error type used by the runtime apis.
 	type Error: std::fmt::Debug + From<String>;
+}
+
+/// Extends the runtime api implementation with some common functionality.
+#[cfg(feature = "std")]
+pub trait ApiExt<Block: BlockT>: ApiErrorExt {
+	/// The state backend that is used to store the block states.
+	type StateBackend: StateBackend<HasherFor<Block>>;
 
 	/// The given closure will be called with api instance. Inside the closure any api call is
 	/// allowed. After doing the api call, the closure is allowed to map the `Result` to a
@@ -250,15 +288,15 @@ pub trait ApiExt<Block: BlockT> {
 	/// On `Ok`, the structure commits the changes to an internal buffer.
 	fn map_api_result<F: FnOnce(&Self) -> result::Result<R, E>, R, E>(
 		&self,
-		map_call: F
+		map_call: F,
 	) -> result::Result<R, E> where Self: Sized;
 
 	/// Checks if the given api is implemented and versions match.
 	fn has_api<A: RuntimeApiInfo + ?Sized>(
 		&self,
-		at: &BlockId<Block>
+		at: &BlockId<Block>,
 	) -> Result<bool, Self::Error> where Self: Sized {
-		self.runtime_version_at(at).map(|v| v.has_api::<A>())
+		self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, |v| v == A::VERSION))
 	}
 
 	/// Check if the given api is implemented and the version passes a predicate.
@@ -267,7 +305,7 @@ pub trait ApiExt<Block: BlockT> {
 		at: &BlockId<Block>,
 		pred: P,
 	) -> Result<bool, Self::Error> where Self: Sized {
-		self.runtime_version_at(at).map(|v| v.has_api_with::<A, _>(pred))
+		self.runtime_version_at(at).map(|v| v.has_api_with(&A::ID, pred))
 	}
 
 	/// Returns the runtime version at the given block id.
@@ -277,8 +315,22 @@ pub trait ApiExt<Block: BlockT> {
 	fn record_proof(&mut self);
 
 	/// Extract the recorded proof.
+	///
 	/// This stops the proof recording.
+	///
+	/// If `record_proof` was not called before, this will return `None`.
 	fn extract_proof(&mut self) -> Option<StorageProof>;
+
+	/// Convert the api object into the storage changes that were done while executing runtime
+	/// api functions.
+	///
+	/// After executing this function, all collected changes are reset.
+	fn into_storage_changes<T: ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>>(
+		&self,
+		backend: &Self::StateBackend,
+		changes_trie_storage: Option<&T>,
+		parent_hash: Block::Hash,
+	) -> Result<StorageChanges<Self::StateBackend, Block>, String> where Self: Sized;
 }
 
 /// Before calling any runtime api function, the runtime need to be initialized
@@ -301,12 +353,44 @@ pub enum InitializeBlock<'a, Block: BlockT> {
 	Do(&'a RefCell<Option<BlockId<Block>>>),
 }
 
-/// Something that can call into the runtime at a given block.
+/// Parameters for [`CallApiAt::call_api_at`].
 #[cfg(feature = "std")]
-pub trait CallRuntimeAt<Block: BlockT> {
-	/// Error type used by the interface.
+pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HasherFor<Block>>> {
+	/// A reference to something that implements the [`Core`] api.
+	pub core_api: &'a C,
+	/// The block id that determines the state that should be setup when calling the function.
+	pub at: &'a BlockId<Block>,
+	/// The name of the function that should be called.
+	pub function: &'static str,
+	/// An optional native call that calls the `function`. This is an optimization to call into a
+	/// native runtime without requiring to encode/decode the parameters. The native runtime can
+	/// still be called when this value is `None`, we then just fallback to encoding/decoding the
+	/// parameters.
+	pub native_call: Option<NC>,
+	/// The encoded arguments of the function.
+	pub arguments: Vec<u8>,
+	/// The overlayed changes that are on top of the state.
+	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
+	/// The cache for storage transactions.
+	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
+	/// Determines if the function requires that `initialize_block` should be called before calling
+	/// the actual function.
+	pub initialize_block: InitializeBlock<'a, Block>,
+	/// The context this function is executed in.
+	pub context: ExecutionContext,
+	/// The optional proof recorder for recording storage accesses.
+	pub recorder: &'a Option<ProofRecorder<Block>>,
+}
+
+/// Something that can call into the an api at a given block.
+#[cfg(feature = "std")]
+pub trait CallApiAt<Block: BlockT> {
+	/// Error type used by the implementation.
 	type Error: std::fmt::Debug + From<String>;
 
+	/// The state backend that is used to store the block states.
+	type StateBackend: StateBackend<HasherFor<Block>>;
+
 	/// Calls the given api function with the given encoded arguments at the given block and returns
 	/// the encoded result.
 	fn call_api_at<
@@ -316,26 +400,66 @@ pub trait CallRuntimeAt<Block: BlockT> {
 		C: Core<Block, Error = Self::Error>,
 	>(
 		&self,
-		core_api: &C,
-		at: &BlockId<Block>,
-		function: &'static str,
-		args: Vec<u8>,
-		changes: &RefCell<OverlayedChanges>,
-		initialize_block: InitializeBlock<'a, Block>,
-		native_call: Option<NC>,
-		context: ExecutionContext,
-		recorder: &Option<ProofRecorder<Block>>,
+		params: CallApiAtParams<'a, Block, C, NC, Self::StateBackend>,
 	) -> Result<NativeOrEncoded<R>, Self::Error>;
 
 	/// Returns the runtime version at the given block.
 	fn runtime_version_at(&self, at: &BlockId<Block>) -> Result<RuntimeVersion, Self::Error>;
 }
 
+/// Auxiliary wrapper that holds an api instance and binds it to the given lifetime.
+#[cfg(feature = "std")]
+pub struct ApiRef<'a, T>(T, std::marker::PhantomData<&'a ()>);
+
+#[cfg(feature = "std")]
+impl<'a, T> From<T> for ApiRef<'a, T> {
+	fn from(api: T) -> Self {
+		ApiRef(api, Default::default())
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'a, T> std::ops::Deref for ApiRef<'a, T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+
+#[cfg(feature = "std")]
+impl<'a, T> std::ops::DerefMut for ApiRef<'a, T> {
+	fn deref_mut(&mut self) -> &mut T {
+		&mut self.0
+	}
+}
+
+/// Something that provides a runtime api.
+#[cfg(feature = "std")]
+pub trait ProvideRuntimeApi<Block: BlockT> {
+	/// The concrete type that provides the api.
+	type Api: ApiExt<Block>;
+
+	/// Returns the runtime api.
+	/// The returned instance will keep track of modifications to the storage. Any successful
+	/// call to an api function, will `commit` its changes to an internal buffer. Otherwise,
+	/// the modifications will be `discarded`. The modifications will not be applied to the
+	/// storage, even on a `commit`.
+	fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api>;
+}
+
+/// Something that provides information about a runtime api.
+#[cfg(feature = "std")]
+pub trait RuntimeApiInfo {
+	/// The identifier of the runtime api.
+	const ID: [u8; 8];
+	/// The version of the runtime api.
+	const VERSION: u32;
+}
+
 /// Extracts the `Api::Error` for a type that provides a runtime api.
 #[cfg(feature = "std")]
-pub type ApiErrorFor<T, Block> = <
-	<T as sp_runtime::traits::ProvideRuntimeApi>::Api as ApiExt<Block>
->::Error;
+pub type ApiErrorFor<T, Block> = <<T as ProvideRuntimeApi<Block>>::Api as ApiErrorExt>::Error;
 
 decl_runtime_apis! {
 	/// The `Core` runtime api that every Substrate runtime needs to implement.

substrate/primitives/api/test/benches/bench.rs

@@ -19,8 +19,9 @@ use substrate_test_runtime_client::{
 	DefaultTestClientBuilderExt, TestClientBuilder,
 	TestClientBuilderExt, runtime::TestAPI,
 };
-use sp_runtime::{generic::BlockId, traits::ProvideRuntimeApi};
+use sp_runtime::generic::BlockId;
 use sp_state_machine::ExecutionStrategy;
+use sp_api::ProvideRuntimeApi;
 
 fn sp_api_benchmark(c: &mut Criterion) {
 	c.bench_function("add one with same runtime api", |b| {

substrate/primitives/api/test/tests/decl_and_impl.rs

@@ -90,16 +90,16 @@ type TestClient = substrate_test_runtime_client::sc_client::Client<
 
 #[test]
 fn test_client_side_function_signature() {
-	let _test: fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>, u64) -> Result<()> =
-		RuntimeApiImpl::<TestClient>::test;
+	let _test: fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>, u64) -> Result<()> =
+		RuntimeApiImpl::<Block, TestClient>::test;
 	let _something_with_block:
-		fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>, Block) -> Result<Block> =
-			RuntimeApiImpl::<TestClient>::something_with_block;
+		fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>, Block) -> Result<Block> =
+			RuntimeApiImpl::<Block, TestClient>::something_with_block;
 
 	#[allow(deprecated)]
 	let _same_name_before_version_2:
-		fn(&RuntimeApiImpl<TestClient>, &BlockId<Block>) -> Result<String> =
-			RuntimeApiImpl::<TestClient>::same_name_before_version_2;
+		fn(&RuntimeApiImpl<Block, TestClient>, &BlockId<Block>) -> Result<String> =
+			RuntimeApiImpl::<Block, TestClient>::same_name_before_version_2;
 }
 
 #[test]

substrate/primitives/api/test/tests/runtime_calls.rs

@@ -14,15 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
+use sp_api::ProvideRuntimeApi;
 use substrate_test_runtime_client::{
 	prelude::*,
 	DefaultTestClientBuilderExt, TestClientBuilder,
 	runtime::{TestAPI, DecodeFails, Transfer, Header},
 };
-use sp_runtime::{
-	generic::BlockId,
-	traits::{ProvideRuntimeApi, Header as HeaderT, Hash as HashT},
-};
+use sp_runtime::{generic::BlockId, traits::{Header as HeaderT, Hash as HashT}};
 use sp_state_machine::{
 	ExecutionStrategy, create_proof_check_backend,
 	execution_proof_check_on_trie_backend,
@@ -174,10 +172,10 @@ fn record_proof_works() {
 
 	// Build the block and record proof
 	let mut builder = client
-		.new_block_at_with_proof_recording(&block_id, Default::default())
+		.new_block_at(&block_id, Default::default(), true)
 		.expect("Creates block builder");
 	builder.push(transaction.clone()).unwrap();
-	let (block, proof) = builder.bake_and_extract_proof().expect("Bake block");
+	let (block, _, proof) = builder.build().expect("Bake block").into_inner();
 
 	let backend = create_proof_check_backend::<<<Header as HeaderT>::Hashing as HashT>::Hasher>(
 		storage_root,

substrate/primitives/api/test/tests/ui/declaring_old_block.stderr

@@ -1,15 +1,15 @@
-error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro!
- --> $DIR/declaring_old_block.rs:4:16
-  |
-4 |     pub trait Api<Block: BlockT> {
-  |                   ^^^^^
-
 error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro! If you try to use a different trait than the substrate `Block` trait, please rename it locally.
  --> $DIR/declaring_old_block.rs:4:23
   |
 4 |     pub trait Api<Block: BlockT> {
   |                          ^^^^^^
 
+error: `Block: BlockT` generic parameter will be added automatically by the `decl_runtime_apis!` macro!
+ --> $DIR/declaring_old_block.rs:4:16
+  |
+4 |     pub trait Api<Block: BlockT> {
+  |                   ^^^^^
+
 warning: unused import: `sp_runtime::traits::Block as BlockT`
  --> $DIR/declaring_old_block.rs:1:5
   |

substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs

@@ -20,7 +20,7 @@ sp_api::impl_runtime_apis! {
 	}
 
 	impl sp_api::Core<Block> for Runtime {
-		fn version() -> runtime_api::RuntimeVersion {
+		fn version() -> sp_api::RuntimeVersion {
 			unimplemented!()
 		}
 		fn execute_block(_: Block) {

substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.stderr

@@ -1,9 +1,3 @@
-error[E0433]: failed to resolve: use of undeclared type or module `runtime_api`
-  --> $DIR/impl_incorrect_method_signature.rs:23:19
-   |
-23 |         fn version() -> runtime_api::RuntimeVersion {
-   |                         ^^^^^^^^^^^ use of undeclared type or module `runtime_api`
-
 error[E0053]: method `test` has an incompatible type for trait
   --> $DIR/impl_incorrect_method_signature.rs:19:17
    |
@@ -39,8 +33,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr
 33 | | }
    | |_- in this macro invocation
    |
-   = note: expected type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
-              found type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<std::string::String>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
+   = note: expected type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
+              found type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<std::string::String>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
 
 error[E0308]: mismatched types
   --> $DIR/impl_incorrect_method_signature.rs:17:1

substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.rs

@@ -17,7 +17,7 @@ sp_api::decl_runtime_apis! {
 mod second {
 	use super::*;
 
-	decl_runtime_apis! {
+	sp_api::decl_runtime_apis! {
 		pub trait Api {
 			fn test2(data: u64);
 		}

substrate/primitives/api/test/tests/ui/impl_two_traits_with_same_name.stderr

@@ -4,74 +4,10 @@ error: Two traits with the same name detected! The trait name is used to generat
 32 |     impl second::Api<Block> for Runtime {
    |                  ^^^
 
-error: cannot find macro `decl_runtime_apis` in this scope
-  --> $DIR/impl_two_traits_with_same_name.rs:20:2
+warning: unused import: `super::*`
+  --> $DIR/impl_two_traits_with_same_name.rs:18:6
    |
-20 |     decl_runtime_apis! {
-   |     ^^^^^^^^^^^^^^^^^
-
-error[E0425]: cannot find function `test2_call_api_at` in `second::runtime_decl_for_Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
+18 |     use super::*;
+   |         ^^^^^^^^
    |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | | ^
-   | | |
-   | |_not found in `second::runtime_decl_for_Api`
-   |   in this macro invocation
-
-error[E0425]: cannot find function `test2_native_call_generator` in `second::runtime_decl_for_Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
-   |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | | ^
-   | | |
-   | |_not found in `second::runtime_decl_for_Api`
-   |   in this macro invocation
-
-error[E0576]: cannot find method or associated constant `test2` in `second::runtime_decl_for_Api::Api`
-  --> $DIR/impl_two_traits_with_same_name.rs:33:6
-   |
-33 |         fn test2(data: u64) {}
-   |            ^^^^^ not found in `second::runtime_decl_for_Api::Api`
-
-error[E0603]: module `runtime_decl_for_Api` is private
-  --> $DIR/impl_two_traits_with_same_name.rs:27:1
-   |
-27 | / sp_api::impl_runtime_apis! {
-28 | |     impl self::Api<Block> for Runtime {
-29 | |         fn test(data: u64) {}
-30 | |     }
-...  |
-34 | |     }
-35 | | }
-   | |_^
-
-error[E0119]: conflicting implementations of trait `runtime_decl_for_Api::Api<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>` for type `Runtime`:
-  --> $DIR/impl_two_traits_with_same_name.rs:32:2
-   |
-28 |     impl self::Api<Block> for Runtime {
-   |     --------------------------------- first implementation here
-...
-32 |     impl second::Api<Block> for Runtime {
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ conflicting implementation for `Runtime`
-
-error[E0119]: conflicting implementations of trait `Api<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>` for type `RuntimeApiImpl<_>`:
-  --> $DIR/impl_two_traits_with_same_name.rs:32:2
-   |
-28 |     impl self::Api<Block> for Runtime {
-   |     --------------------------------- first implementation here
-...
-32 |     impl second::Api<Block> for Runtime {
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ conflicting implementation for `RuntimeApiImpl<_>`
+   = note: `#[warn(unused_imports)]` on by default

substrate/primitives/api/test/tests/ui/invalid_api_version.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version.rs:2:4
   |

substrate/primitives/api/test/tests/ui/invalid_api_version_2.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version_2.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version("1")]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version_2.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version("1")]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version_2.rs:2:4
   |

substrate/primitives/api/test/tests/ui/invalid_api_version_3.stderr

@@ -1,31 +1,3 @@
-error: can't qualify macro invocation with `pub`
- --> $DIR/invalid_api_version_3.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version()]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-  |
-  = help: try adjusting the macro to put `pub` inside the invocation
-
-error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
- --> $DIR/invalid_api_version_3.rs:1:1
-  |
-1 | / sp_api::decl_runtime_apis! {
-2 | |     #[api_version()]
-3 | |     pub trait Api {
-4 | |         fn test(data: u64);
-5 | |     }
-6 | | }
-  | | ^ in this macro invocation
-  | |_|
-  | 
-
 error: Unexpected `api_version` attribute. The supported format is `api_version(1)`
  --> $DIR/invalid_api_version_3.rs:2:4
   |

substrate/primitives/api/test/tests/ui/missing_block_generic_parameter.stderr

@@ -3,9 +3,3 @@ error: Missing `Block` generic parameter.
    |
 18 |     impl self::Api for Runtime {
    |                ^^^
-
-error[E0107]: wrong number of type arguments: expected 1, found 0
-  --> $DIR/missing_block_generic_parameter.rs:18:7
-   |
-18 |     impl self::Api for Runtime {
-   |          ^^^^^^^^^ expected 1 type argument

substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs

@@ -22,7 +22,7 @@ sp_api::impl_runtime_apis! {
 	}
 
 	impl sp_api::Core<Block> for Runtime {
-		fn version() -> runtime_api::RuntimeVersion {
+		fn version() -> sp_api::RuntimeVersion {
 			unimplemented!()
 		}
 		fn execute_block(_: Block) {

substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr

@@ -1,9 +1,3 @@
-error[E0433]: failed to resolve: use of undeclared type or module `runtime_api`
-  --> $DIR/type_reference_in_impl_runtime_apis_call.rs:25:19
-   |
-25 |         fn version() -> runtime_api::RuntimeVersion {
-   |                         ^^^^^^^^^^^ use of undeclared type or module `runtime_api`
-
 error[E0053]: method `test` has an incompatible type for trait
   --> $DIR/type_reference_in_impl_runtime_apis_call.rs:19:17
    |
@@ -39,8 +33,8 @@ error[E0053]: method `Api_test_runtime_api_impl` has an incompatible type for tr
 35 | | }
    | |_- in this macro invocation
    |
-   = note: expected type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
-              found type `fn(&RuntimeApiImpl<RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<&u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallRuntimeAt<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u64, sp_runtime::traits::BlakeTwo256>, substrate_test_runtime::Extrinsic>>>::Error>`
+   = note: expected type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
+              found type `fn(&RuntimeApiImpl<__SR_API_BLOCK__, RuntimeApiImplCall>, &sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::BlockId<__SR_API_BLOCK__>, sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::ExecutionContext, std::option::Option<&u64>, std::vec::Vec<u8>) -> std::result::Result<sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::NativeOrEncoded<()>, <RuntimeApiImplCall as sp_api_hidden_includes_DECL_RUNTIME_APIS::sp_api::CallApiAt<__SR_API_BLOCK__>>::Error>`
 
 error[E0308]: mismatched types
   --> $DIR/type_reference_in_impl_runtime_apis_call.rs:17:1