Do not call initialize_block before any runtime api (#8953)

* Do not call `initialize_block` before any runtime api

Before this change we always called `initialize_block` before calling
into the runtime. There was already support with `skip_initialize` to skip
the initialization. Almost no runtime_api requires that
`initialize_block` is called before. Actually this only leads to higher
execution times most of the time, because all runtime modules are
initialized and this is especially expensive when the block contained a
runtime upgrade.

TLDR: Do not call `initialize_block` before calling a runtime api.

* Change `validate_transaction` interface

* Fix rpc test

* Fixes and comments

* Some docs

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -58,21 +58,9 @@ const CHANGED_IN_ATTRIBUTE: &str = "changed_in";
 ///
 /// Is used when a trait method was renamed.
 const RENAMED_ATTRIBUTE: &str = "renamed";
-/// The `skip_initialize_block` attribute.
-///
-/// Is used when a trait method does not require that the block is initialized
-/// before being called.
-const SKIP_INITIALIZE_BLOCK_ATTRIBUTE: &str = "skip_initialize_block";
-/// The `initialize_block` attribute.
-///
-/// A trait method tagged with this attribute, initializes the runtime at
-/// certain block.
-const INITIALIZE_BLOCK_ATTRIBUTE: &str = "initialize_block";
 /// All attributes that we support in the declaration of a runtime api trait.
 const SUPPORTED_ATTRIBUTE_NAMES: &[&str] = &[
-	CORE_TRAIT_ATTRIBUTE, API_VERSION_ATTRIBUTE, CHANGED_IN_ATTRIBUTE,
-	RENAMED_ATTRIBUTE, SKIP_INITIALIZE_BLOCK_ATTRIBUTE,
-	INITIALIZE_BLOCK_ATTRIBUTE,
+	CORE_TRAIT_ATTRIBUTE, API_VERSION_ATTRIBUTE, CHANGED_IN_ATTRIBUTE, RENAMED_ATTRIBUTE,
 ];
 
 /// The structure used for parsing the runtime api declarations.
@@ -376,15 +364,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 			continue;
 		}
 
-		let skip_initialize_block = attrs.contains_key(SKIP_INITIALIZE_BLOCK_ATTRIBUTE);
-		let update_initialized_block = if attrs.contains_key(INITIALIZE_BLOCK_ATTRIBUTE) {
-			quote!(
-				|| *initialized_block.borrow_mut() = Some(*at)
-			)
-		} else {
-			quote!(|| ())
-		};
-
 		// Parse the renamed attributes.
 		let mut renames = Vec::new();
 		if let Some((_, a)) = attrs
@@ -413,72 +392,54 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				NC: FnOnce() -> std::result::Result<R, #crate_::ApiError> + std::panic::UnwindSafe,
 				Block: #crate_::BlockT,
 				T: #crate_::CallApiAt<Block>,
-				C: #crate_::Core<Block>,
 			>(
 				call_runtime_at: &T,
-				core_api: &C,
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
 				storage_transaction_cache: &std::cell::RefCell<
 					#crate_::StorageTransactionCache<Block, T::StateBackend>
 				>,
-				initialized_block: &std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 				native_call: Option<NC>,
 				context: #crate_::ExecutionContext,
 				recorder: &Option<#crate_::ProofRecorder<Block>>,
 			) -> std::result::Result<#crate_::NativeOrEncoded<R>, #crate_::ApiError> {
 				let version = call_runtime_at.runtime_version_at(at)?;
-				use #crate_::InitializeBlock;
-				let initialize_block = if #skip_initialize_block {
-					InitializeBlock::Skip
-				} else {
-					InitializeBlock::Do(&initialized_block)
-				};
-				let update_initialized_block = #update_initialized_block;
 
 				#(
 					// Check if we need to call the function by an old name.
 					if version.apis.iter().any(|(s, v)| {
 						s == &ID && *v < #versions
 					}) {
-						let params = #crate_::CallApiAtParams::<_, _, fn() -> _, _> {
-							core_api,
+						let params = #crate_::CallApiAtParams::<_, fn() -> _, _> {
 							at,
 							function: #old_names,
 							native_call: None,
 							arguments: args,
 							overlayed_changes: changes,
 							storage_transaction_cache,
-							initialize_block,
 							context,
 							recorder,
 						};
 
 						let ret = call_runtime_at.call_api_at(params)?;
 
-						update_initialized_block();
 						return Ok(ret)
 					}
 				)*
 
 				let params = #crate_::CallApiAtParams {
-					core_api,
 					at,
 					function: #trait_fn_name,
 					native_call,
 					arguments: args,
 					overlayed_changes: changes,
 					storage_transaction_cache,
-					initialize_block,
 					context,
 					recorder,
 				};
 
-				let ret = call_runtime_at.call_api_at(params)?;
-
-				update_initialized_block();
-				Ok(ret)
+				call_runtime_at.call_api_at(params)
 			}
 		));
 	}

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -122,9 +122,9 @@ fn generate_impl_calls(
 
 				impl_calls.push((
 					impl_trait_ident.clone(),
-					 method.sig.ident.clone(),
-					 impl_call,
-					 filter_cfg_attrs(&impl_.attrs),
+					method.sig.ident.clone(),
+					impl_call,
+					filter_cfg_attrs(&impl_.attrs),
 				));
 			}
 		}
@@ -186,7 +186,7 @@ fn generate_wasm_interface(impls: &[ItemImpl]) -> Result<TokenStream> {
 
 					#c::init_runtime_logger();
 
-					let output = { #impl_ };
+					let output = (move || { #impl_ })();
 					#c::to_substrate_wasm_fn_return_value(&output)
 				}
 			)
@@ -205,7 +205,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 		pub struct RuntimeApiImpl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block> + 'static> {
 			call: &'static C,
 			commit_on_success: std::cell::RefCell<bool>,
-			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
 			storage_transaction_cache: std::cell::RefCell<
 				#crate_::StorageTransactionCache<Block, C::StateBackend>
@@ -265,6 +264,15 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					.map(|v| v.has_api_with(&A::ID, pred))
 			}
 
+			fn api_version<A: #crate_::RuntimeApiInfo + ?Sized>(
+				&self,
+				at: &#crate_::BlockId<Block>,
+			) -> std::result::Result<Option<u32>, #crate_::ApiError> where Self: Sized {
+				self.call
+					.runtime_version_at(at)
+					.map(|v| v.api_version(&A::ID))
+			}
+
 			fn record_proof(&mut self) {
 				self.recorder = Some(Default::default());
 			}
@@ -291,7 +299,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 				#crate_::StorageChanges<C::StateBackend, Block>,
 				String
 			> where Self: Sized {
-				self.initialized_block.borrow_mut().take();
 				self.changes.replace(Default::default()).into_storage_changes(
 					backend,
 					changes_trie_state,
@@ -315,7 +322,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 				RuntimeApiImpl {
 					call: unsafe { std::mem::transmute(call) },
 					commit_on_success: true.into(),
-					initialized_block: None.into(),
 					changes: Default::default(),
 					recorder: Default::default(),
 					storage_transaction_cache: Default::default(),
@@ -329,10 +335,8 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 				R: #crate_::Encode + #crate_::Decode + PartialEq,
 				F: FnOnce(
 					&C,
-					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
 					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
-					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 					&Option<#crate_::ProofRecorder<Block>>,
 				) -> std::result::Result<#crate_::NativeOrEncoded<R>, E>,
 				E,
@@ -345,10 +349,8 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 				}
 				let res = call_api_at(
 					&self.call,
-					self,
 					&self.changes,
 					&self.storage_transaction_cache,
-					&self.initialized_block,
 					&self.recorder,
 				);
 
@@ -501,20 +503,16 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 					self.call_api_at(
 						|
 							call_runtime_at,
-							core_api,
 							changes,
 							storage_transaction_cache,
-							initialized_block,
 							recorder
 						| {
 							#runtime_mod_path #call_api_at_call(
 								call_runtime_at,
-								core_api,
 								at,
 								params_encoded,
 								changes,
 								storage_transaction_cache,
-								initialized_block,
 								params.map(|p| {
 									#runtime_mod_path #native_call_generator_ident ::
 										<#runtime, __SR_API_BLOCK__ #(, #trait_generic_arguments )*> (

substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs

@@ -94,6 +94,13 @@ fn implement_common_api_traits(
 				Ok(pred(A::VERSION))
 			}
 
+			fn api_version<A: #crate_::RuntimeApiInfo + ?Sized>(
+				&self,
+				_: &#crate_::BlockId<#block_type>,
+			) -> std::result::Result<Option<u32>, #crate_::ApiError> where Self: Sized {
+				Ok(Some(A::VERSION))
+			}
+
 			fn record_proof(&mut self) {
 				unimplemented!("`record_proof` not implemented for runtime api mocks")
 			}

substrate/primitives/api/src/lib.rs

@@ -17,20 +17,29 @@
 
 //! Substrate runtime api
 //!
-//! The Substrate runtime api is the crucial interface between the node and the runtime.
-//! Every call that goes into the runtime is done with a runtime api. The runtime apis are not fixed.
-//! Every Substrate user can define its own apis with
-//! [`decl_runtime_apis`](macro.decl_runtime_apis.html) and implement them in
-//! the runtime with [`impl_runtime_apis`](macro.impl_runtime_apis.html).
+//! The Substrate runtime api is the interface between the node and the runtime. There isn't a fixed
+//! set of runtime apis, instead it is up to the user to declare and implement these runtime apis.
+//! The declaration of a runtime api is normally done outside of a runtime, while the implementation
+//! of it has to be done in the runtime. We provide the [`decl_runtime_apis!`] macro for declaring
+//! a runtime api and the [`impl_runtime_apis!`] for implementing them. The macro docs provide more
+//! information on how to use them and what kind of attributes we support.
 //!
-//! Every Substrate runtime needs to implement the [`Core`] runtime api. This api provides the basic
-//! functionality that every runtime needs to export.
+//! It is required that each runtime implements at least the [`Core`] runtime api. This runtime api
+//! provides all the core functions that Substrate expects from a runtime.
 //!
-//! Besides the macros and the [`Core`] runtime api, this crates provides the [`Metadata`] runtime
-//! api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi`] trait.
+//! # Versioning
 //!
-//! On a meta level this implies, the client calls the generated API from the client perspective.
+//! Runtime apis support versioning. Each runtime api itself has a version attached. It is also
+//! supported to change function signatures or names in a non-breaking way. For more information on
+//! versioning check the [`decl_runtime_apis!`] macro.
 //!
+//! All runtime apis and their versions are returned as part of the [`RuntimeVersion`]. This can be
+//! used to check which runtime api version is currently provided by the on-chain runtime.
+//!
+//! # Testing
+//!
+//! For testing we provide the [`mock_impl_runtime_apis!`] macro that lets you implement a runtime
+//! api for a mocked object to use it in tests.
 //!
 //! # Logging
 //!
@@ -43,6 +52,17 @@
 //! that this feature instructs `log` and `tracing` to disable logging at compile time by setting
 //! the `max_level_off` feature for these crates. So, you should not enable this feature for a
 //! native build as otherwise the node will not output any log messages.
+//!
+//! # How does it work?
+//!
+//! Each runtime api is declared as a trait with functions. When compiled to WASM, each implemented
+//! runtime api function is exported as a function with the following naming scheme
+//! `${TRAIT_NAME}_${FUNCTION_NAME}`. Such a function has the following signature
+//! `(ptr: *u8, length: u32) -> u64`. It takes a pointer to an `u8` array and its length as an
+//! argument. This `u8` array is expected to be the SCALE encoded parameters of the function as
+//! defined in the trait. The return value is an `u64` that represents `length << 32 | pointer` of an
+//! `u8` array. This return value `u8` array contains the SCALE encoded return value as defined by
+//! the trait function. The macros take care to encode the parameters and to decode the return value.
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
@@ -99,7 +119,7 @@ pub const MAX_EXTRINSIC_DEPTH: u32 = 256;
 /// to the client side and the runtime side. This generic parameter is usable by the user.
 ///
 /// For implementing these macros you should use the
-/// [`impl_runtime_apis!`](macro.impl_runtime_apis.html) macro.
+/// [`impl_runtime_apis!`] macro.
 ///
 /// # Example
 ///
@@ -461,6 +481,12 @@ pub trait ApiExt<Block: BlockT> {
 		pred: P,
 	) -> Result<bool, ApiError> where Self: Sized;
 
+	/// Returns the version of the given api.
+	fn api_version<A: RuntimeApiInfo + ?Sized>(
+		&self,
+		at: &BlockId<Block>,
+	) -> Result<Option<u32>, ApiError> where Self: Sized;
+
 	/// Start recording all accessed trie nodes for generating proofs.
 	fn record_proof(&mut self);
 
@@ -489,31 +515,9 @@ pub trait ApiExt<Block: BlockT> {
 	> where Self: Sized;
 }
 
-/// Before calling any runtime api function, the runtime need to be initialized
-/// at the requested block. However, some functions like `execute_block` or
-/// `initialize_block` itself don't require to have the runtime initialized
-/// at the requested block.
-///
-/// `call_api_at` is instructed by this enum to do the initialization or to skip
-/// it.
-#[cfg(feature = "std")]
-#[derive(Clone, Copy)]
-pub enum InitializeBlock<'a, Block: BlockT> {
-	/// Skip initializing the runtime for a given block.
-	///
-	/// This is used by functions who do the initialization by themselves or don't require it.
-	Skip,
-	/// Initialize the runtime for a given block.
-	///
-	/// If the stored `BlockId` is `Some(_)`, the runtime is currently initialized at this block.
-	Do(&'a RefCell<Option<BlockId<Block>>>),
-}
-
 /// Parameters for [`CallApiAt::call_api_at`].
 #[cfg(feature = "std")]
-pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HashFor<Block>>> {
-	/// A reference to something that implements the [`Core`] api.
-	pub core_api: &'a C,
+pub struct CallApiAtParams<'a, Block: BlockT, NC, Backend: StateBackend<HashFor<Block>>> {
 	/// The block id that determines the state that should be setup when calling the function.
 	pub at: &'a BlockId<Block>,
 	/// The name of the function that should be called.
@@ -529,9 +533,6 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HashF
 	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
 	/// The cache for storage transactions.
 	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
-	/// Determines if the function requires that `initialize_block` should be called before calling
-	/// the actual function.
-	pub initialize_block: InitializeBlock<'a, Block>,
 	/// The context this function is executed in.
 	pub context: ExecutionContext,
 	/// The optional proof recorder for recording storage accesses.
@@ -550,10 +551,9 @@ pub trait CallApiAt<Block: BlockT> {
 		'a,
 		R: Encode + Decode + PartialEq,
 		NC: FnOnce() -> result::Result<R, ApiError> + UnwindSafe,
-		C: Core<Block>,
 	>(
 		&self,
-		params: CallApiAtParams<'a, Block, C, NC, Self::StateBackend>,
+		params: CallApiAtParams<'a, Block, NC, Self::StateBackend>,
 	) -> Result<NativeOrEncoded<R>, ApiError>;
 
 	/// Returns the runtime version at the given block.
@@ -704,12 +704,9 @@ decl_runtime_apis! {
 		#[changed_in(3)]
 		fn version() -> OldRuntimeVersion;
 		/// Execute the given block.
-		#[skip_initialize_block]
 		fn execute_block(block: Block);
 		/// Initialize a block with the given header.
 		#[renamed("initialise_block", 2)]
-		#[skip_initialize_block]
-		#[initialize_block]
 		fn initialize_block(header: &<Block as BlockT>::Header);
 	}
 

substrate/primitives/api/test/tests/runtime_calls.rs

@@ -15,11 +15,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use sp_api::ProvideRuntimeApi;
+use sp_api::{ProvideRuntimeApi, Core};
 use substrate_test_runtime_client::{
 	prelude::*,
 	DefaultTestClientBuilderExt, TestClientBuilder,
-	runtime::{TestAPI, DecodeFails, Transfer, Block},
+	runtime::{TestAPI, DecodeFails, Transfer, Block, Header},
 };
 use sp_runtime::{generic::BlockId, traits::{Header as HeaderT, HashFor}};
 use sp_state_machine::{
@@ -133,26 +133,13 @@ fn initialize_block_works() {
 	let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build();
 	let runtime_api = client.runtime_api();
 	let block_id = BlockId::Number(client.chain_info().best_number);
+	runtime_api.initialize_block(
+		&block_id,
+		&Header::new(1, Default::default(), Default::default(), Default::default(), Default::default()),
+	).unwrap();
 	assert_eq!(runtime_api.get_block_number(&block_id).unwrap(), 1);
 }
 
-#[test]
-fn initialize_block_is_called_only_once() {
-	let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build();
-	let runtime_api = client.runtime_api();
-	let block_id = BlockId::Number(client.chain_info().best_number);
-	assert_eq!(runtime_api.take_block_number(&block_id).unwrap(), Some(1));
-	assert_eq!(runtime_api.take_block_number(&block_id).unwrap(), None);
-}
-
-#[test]
-fn initialize_block_is_skipped() {
-	let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build();
-	let runtime_api = client.runtime_api();
-	let block_id = BlockId::Number(client.chain_info().best_number);
-	assert!(runtime_api.without_initialize_block(&block_id).unwrap());
-}
-
 #[test]
 fn record_proof_works() {
 	let (client, longest_chain) = TestClientBuilder::new()