Removal of execution strategies (#14387)

* Start

* More work!

* Moar

* More changes

* More fixes

* More worrk

* More fixes

* More fixes to make it compile

* Adds `NoOffchainStorage`

* Pass the extensions

* Small basti making small progress

* Fix merge errors and remove `ExecutionContext`

* Move registration of `ReadRuntimeVersionExt` to `ExecutionExtension`

Instead of registering `ReadRuntimeVersionExt` in `sp-state-machine` it is moved to
`ExecutionExtension` which provides the default extensions.

* Fix compilation

* Register the global extensions inside runtime api instance

* Fixes

* Fix `generate_initial_session_keys` by passing the keystore extension

* Fix the grandpa tests

* Fix more tests

* Fix more tests

* Don't set any heap pages if there isn't an override

* Fix small fallout

* FMT

* Fix tests

* More tests

* Offchain worker custom extensions

* More fixes

* Make offchain tx pool creation reusable

Introduces an `OffchainTransactionPoolFactory` for creating offchain transactions pools that can be
registered in the runtime externalities context. This factory will be required for a later pr to
make the creation of offchain transaction pools easier.

* Fixes

* Fixes

* Set offchain transaction pool in BABE before using it in the runtime

* Add the `offchain_tx_pool` to Grandpa as well

* Fix the nodes

* Print some error when using the old warnings

* Fix merge issues

* Fix compilation

* Rename `babe_link`

* Rename to `offchain_tx_pool_factory`

* Cleanup

* FMT

* Fix benchmark name

* Fix `try-runtime`

* Remove `--execution` CLI args

* Make clippy happy

* Forward bls functions

* Fix docs

* Update UI tests

* Update client/api/src/execution_extensions.rs

Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/cli/src/params/import_params.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/api/src/execution_extensions.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Pass the offchain storage to the MMR RPC

* Update client/api/src/execution_extensions.rs

Co-authored-by: Sebastian Kunert <skunert49@gmail.com>

* Review comments

* Fixes

---------

Co-authored-by: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com>
Co-authored-by: Koute <koute@users.noreply.github.com>
Co-authored-by: Sebastian Kunert <skunert49@gmail.com>
This commit is contained in:
Bastian Köcher
2023-07-11 16:21:38 +02:00
committed by GitHub
parent a2b01c061b
commit 5eb816d7a6
96 changed files with 1175 additions and 1499 deletions
+2
View File
@@ -18,6 +18,7 @@ sp-api-proc-macro = { version = "4.0.0-dev", path = "proc-macro" }
sp-core = { version = "21.0.0", default-features = false, path = "../core" }
sp-std = { version = "8.0.0", default-features = false, path = "../std" }
sp-runtime = { version = "24.0.0", default-features = false, path = "../runtime" }
sp-externalities = { version = "0.19.0", default-features = false, optional = true, path = "../externalities" }
sp-version = { version = "22.0.0", default-features = false, path = "../version" }
sp-state-machine = { version = "0.28.0", default-features = false, optional = true, path = "../state-machine" }
sp-trie = { version = "22.0.0", default-features = false, optional = true, path = "../trie" }
@@ -35,6 +36,7 @@ default = ["std"]
std = [
"codec/std",
"sp-core/std",
"sp-externalities",
"sp-std/std",
"sp-runtime/std",
"sp-state-machine/std",
@@ -315,7 +315,6 @@ impl<'a> ToClientSideDecl<'a> {
fn __runtime_api_internal_call_api_at(
&self,
at: #block_hash,
context: #crate_::ExecutionContext,
params: std::vec::Vec<u8>,
fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str,
) -> std::result::Result<std::vec::Vec<u8>, #crate_::ApiError>;
@@ -335,9 +334,8 @@ impl<'a> ToClientSideDecl<'a> {
items.into_iter().for_each(|i| match i {
TraitItem::Fn(method) => {
let (fn_decl, fn_decl_ctx) = self.fold_trait_item_fn(method, trait_generics_num);
let fn_decl = self.create_method_decl(method, trait_generics_num);
result.push(fn_decl.into());
result.push(fn_decl_ctx.into());
},
r => result.push(r),
});
@@ -345,41 +343,12 @@ impl<'a> ToClientSideDecl<'a> {
result
}
fn fold_trait_item_fn(
&mut self,
method: TraitItemFn,
trait_generics_num: usize,
) -> (TraitItemFn, TraitItemFn) {
let crate_ = self.crate_;
let context = quote!( #crate_::ExecutionContext::OffchainCall(None) );
let fn_decl = self.create_method_decl(method.clone(), context, trait_generics_num);
let fn_decl_ctx = self.create_method_decl_with_context(method, trait_generics_num);
(fn_decl, fn_decl_ctx)
}
fn create_method_decl_with_context(
&mut self,
method: TraitItemFn,
trait_generics_num: usize,
) -> TraitItemFn {
let crate_ = self.crate_;
let context_arg: syn::FnArg = parse_quote!( context: #crate_::ExecutionContext );
let mut fn_decl_ctx = self.create_method_decl(method, quote!(context), trait_generics_num);
fn_decl_ctx.sig.ident =
Ident::new(&format!("{}_with_context", &fn_decl_ctx.sig.ident), Span::call_site());
fn_decl_ctx.sig.inputs.insert(2, context_arg);
fn_decl_ctx
}
/// Takes the method declared by the user and creates the declaration we require for the runtime
/// api client side. This method will call by default the `method_runtime_api_impl` for doing
/// the actual call into the runtime.
fn create_method_decl(
&mut self,
mut method: TraitItemFn,
context: TokenStream,
trait_generics_num: usize,
) -> TraitItemFn {
let params = match extract_parameter_names_types_and_borrows(
@@ -467,7 +436,6 @@ impl<'a> ToClientSideDecl<'a> {
<Self as #trait_name<#( #underscores ),*>>::__runtime_api_internal_call_api_at(
self,
__runtime_api_at_param__,
#context,
__runtime_api_impl_params_encoded__,
&|_version| {
#(
@@ -224,8 +224,8 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
Ok(quote!(
pub struct RuntimeApi {}
/// Implements all runtime apis for the client side.
#crate_::std_enabled! {
/// Implements all runtime apis for the client side.
pub struct RuntimeApiImpl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block> + 'static> {
call: &'static C,
transaction_depth: std::cell::RefCell<u16>,
@@ -234,6 +234,9 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
#crate_::StorageTransactionCache<Block, C::StateBackend>
>,
recorder: std::option::Option<#crate_::ProofRecorder<Block>>,
call_context: #crate_::CallContext,
extensions: std::cell::RefCell<#crate_::Extensions>,
extensions_generated_for: std::cell::RefCell<std::option::Option<Block::Hash>>,
}
impl<Block: #crate_::BlockT, C: #crate_::CallApiAt<Block>> #crate_::ApiExt<Block> for
@@ -321,6 +324,14 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
state_version,
)
}
fn set_call_context(&mut self, call_context: #crate_::CallContext) {
self.call_context = call_context;
}
fn register_extension<E: #crate_::Extension>(&mut self, extension: E) {
std::cell::RefCell::borrow_mut(&self.extensions).register(extension);
}
}
impl<Block: #crate_::BlockT, C> #crate_::ConstructRuntimeApi<Block, C>
@@ -339,6 +350,9 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
changes: std::default::Default::default(),
recorder: std::default::Default::default(),
storage_transaction_cache: std::default::Default::default(),
call_context: #crate_::CallContext::Offchain,
extensions: std::default::Default::default(),
extensions_generated_for: std::default::Default::default(),
}.into()
}
}
@@ -480,7 +494,6 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
fn __runtime_api_internal_call_api_at(
&self,
at: <__SrApiBlock__ as #crate_::BlockT>::Hash,
context: #crate_::ExecutionContext,
params: std::vec::Vec<u8>,
fn_name: &dyn Fn(#crate_::RuntimeVersion) -> &'static str,
) -> std::result::Result<std::vec::Vec<u8>, #crate_::ApiError> {
@@ -498,14 +511,34 @@ impl<'a> ApiRuntimeImplToApiRuntimeApiImpl<'a> {
at,
)?;
match &mut *std::cell::RefCell::borrow_mut(&self.extensions_generated_for) {
Some(generated_for) => {
if *generated_for != at {
return std::result::Result::Err(
#crate_::ApiError::UsingSameInstanceForDifferentBlocks
)
}
},
generated_for @ None => {
#crate_::CallApiAt::<__SrApiBlock__>::initialize_extensions(
self.call,
at,
&mut std::cell::RefCell::borrow_mut(&self.extensions),
)?;
*generated_for = Some(at);
}
}
let params = #crate_::CallApiAtParams {
at,
function: (*fn_name)(version),
arguments: params,
overlayed_changes: &self.changes,
storage_transaction_cache: &self.storage_transaction_cache,
context,
call_context: self.call_context,
recorder: &self.recorder,
extensions: &self.extensions,
};
#crate_::CallApiAt::<__SrApiBlock__>::call_api_at(
@@ -121,13 +121,20 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result<To
> where Self: Sized {
unimplemented!("`into_storage_changes` not implemented for runtime api mocks")
}
fn set_call_context(&mut self, _: #crate_::CallContext) {
unimplemented!("`set_call_context` not implemented for runtime api mocks")
}
fn register_extension<E: #crate_::Extension>(&mut self, _: E) {
unimplemented!("`register_extension` not implemented for runtime api mocks")
}
}
impl #crate_::Core<#block_type> for #self_ty {
fn __runtime_api_internal_call_api_at(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
_: #crate_::ExecutionContext,
_: std::vec::Vec<u8>,
_: &dyn Fn(#crate_::RuntimeVersion) -> &'static str,
) -> std::result::Result<std::vec::Vec<u8>, #crate_::ApiError> {
@@ -141,14 +148,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result<To
unimplemented!("`Core::version` not implemented for runtime api mocks")
}
fn version_with_context(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
_: #crate_::ExecutionContext,
) -> std::result::Result<#crate_::RuntimeVersion, #crate_::ApiError> {
unimplemented!("`Core::version` not implemented for runtime api mocks")
}
fn execute_block(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
@@ -157,15 +156,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result<To
unimplemented!("`Core::execute_block` not implemented for runtime api mocks")
}
fn execute_block_with_context(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
_: #crate_::ExecutionContext,
_: #block_type,
) -> std::result::Result<(), #crate_::ApiError> {
unimplemented!("`Core::execute_block` not implemented for runtime api mocks")
}
fn initialize_block(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
@@ -173,15 +163,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result<To
) -> std::result::Result<(), #crate_::ApiError> {
unimplemented!("`Core::initialize_block` not implemented for runtime api mocks")
}
fn initialize_block_with_context(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
_: #crate_::ExecutionContext,
_: &<#block_type as #crate_::BlockT>::Header,
) -> std::result::Result<(), #crate_::ApiError> {
unimplemented!("`Core::initialize_block` not implemented for runtime api mocks")
}
}
))
}
@@ -255,26 +236,12 @@ impl<'a> FoldRuntimeApiImpl<'a> {
let crate_ = generate_crate_access();
// We also need to overwrite all the `_with_context` methods. To do this,
// we clone all methods and add them again with the new name plus one more argument.
impl_item.items.extend(impl_item.items.clone().into_iter().filter_map(|i| {
if let syn::ImplItem::Fn(mut m) = i {
m.sig.ident = quote::format_ident!("{}_with_context", m.sig.ident);
m.sig.inputs.insert(2, parse_quote!( _: #crate_::ExecutionContext ));
Some(m.into())
} else {
None
}
}));
let block_type = self.block_type;
impl_item.items.push(parse_quote! {
fn __runtime_api_internal_call_api_at(
&self,
_: <#block_type as #crate_::BlockT>::Hash,
_: #crate_::ExecutionContext,
_: std::vec::Vec<u8>,
_: &dyn Fn(#crate_::RuntimeVersion) -> &'static str,
) -> std::result::Result<std::vec::Vec<u8>, #crate_::ApiError> {
+26 -3
View File
@@ -78,11 +78,17 @@ pub use hash_db::Hasher;
#[doc(hidden)]
pub use scale_info;
#[doc(hidden)]
pub use sp_core::offchain;
#[doc(hidden)]
#[cfg(not(feature = "std"))]
pub use sp_core::to_substrate_wasm_fn_return_value;
#[doc(hidden)]
#[cfg(feature = "std")]
pub use sp_core::traits::CallContext;
use sp_core::OpaqueMetadata;
#[doc(hidden)]
pub use sp_core::{offchain, ExecutionContext};
#[cfg(feature = "std")]
pub use sp_externalities::{Extension, Extensions};
#[doc(hidden)]
#[cfg(feature = "frame-metadata")]
pub use sp_metadata_ir::{self as metadata_ir, frame_metadata as metadata};
@@ -518,6 +524,8 @@ pub enum ApiError {
Application(#[from] Box<dyn std::error::Error + Send + Sync>),
#[error("Api called for an unknown Block: {0}")]
UnknownBlock(String),
#[error("Using the same api instance to call into multiple independent blocks.")]
UsingSameInstanceForDifferentBlocks,
}
/// Extends the runtime api implementation with some common functionality.
@@ -581,6 +589,12 @@ pub trait ApiExt<Block: BlockT> {
) -> Result<StorageChanges<Self::StateBackend, Block>, String>
where
Self: Sized;
/// Set the [`CallContext`] to be used by the runtime api calls done by this instance.
fn set_call_context(&mut self, call_context: CallContext);
/// Register an [`Extension`] that will be accessible while executing a runtime api call.
fn register_extension<E: Extension>(&mut self, extension: E);
}
/// Parameters for [`CallApiAt::call_api_at`].
@@ -596,10 +610,12 @@ pub struct CallApiAtParams<'a, Block: BlockT, Backend: StateBackend<HashFor<Bloc
pub overlayed_changes: &'a RefCell<OverlayedChanges>,
/// The cache for storage transactions.
pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
/// The context this function is executed in.
pub context: ExecutionContext,
/// The call context of this call.
pub call_context: CallContext,
/// The optional proof recorder for recording storage accesses.
pub recorder: &'a Option<ProofRecorder<Block>>,
/// The extensions that should be used for this call.
pub extensions: &'a RefCell<Extensions>,
}
/// Something that can call into the an api at a given block.
@@ -620,6 +636,13 @@ pub trait CallApiAt<Block: BlockT> {
/// Get the state `at` the given block.
fn state_at(&self, at: Block::Hash) -> Result<Self::StateBackend, ApiError>;
/// Initialize the `extensions` for the given block `at` by using the global extensions factory.
fn initialize_extensions(
&self,
at: Block::Hash,
extensions: &mut Extensions,
) -> Result<(), ApiError>;
}
/// Auxiliary wrapper that holds an api instance and binds it to the given lifetime.
@@ -17,7 +17,6 @@
use criterion::{criterion_group, criterion_main, Criterion};
use sp_api::ProvideRuntimeApi;
use sp_state_machine::ExecutionStrategy;
use substrate_test_runtime_client::{
runtime::TestAPI, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt,
};
@@ -56,17 +55,13 @@ fn sp_api_benchmark(c: &mut Criterion) {
});
c.bench_function("calling function by function pointer in wasm", |b| {
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::AlwaysWasm)
.build();
let client = TestClientBuilder::new().build();
let best_hash = client.chain_info().best_hash;
b.iter(|| client.runtime_api().benchmark_indirect_call(best_hash).unwrap())
});
c.bench_function("calling function in wasm", |b| {
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::AlwaysWasm)
.build();
c.bench_function("calling function", |b| {
let client = TestClientBuilder::new().build();
let best_hash = client.chain_info().best_hash;
b.iter(|| client.runtime_api().benchmark_direct_call(best_hash).unwrap())
});
@@ -22,9 +22,8 @@ use sp_runtime::{
traits::{HashFor, Header as HeaderT},
TransactionOutcome,
};
use sp_state_machine::{
create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionStrategy,
};
use sp_state_machine::{create_proof_check_backend, execution_proof_check_on_trie_backend};
use substrate_test_runtime_client::{
prelude::*,
runtime::{Block, Header, TestAPI, Transfer},
@@ -36,29 +35,18 @@ use sc_block_builder::BlockBuilderProvider;
use sp_consensus::SelectChain;
use substrate_test_runtime_client::sc_executor::WasmExecutor;
fn calling_function_with_strat(strat: ExecutionStrategy) {
let client = TestClientBuilder::new().set_execution_strategy(strat).build();
#[test]
fn calling_runtime_function() {
let client = TestClientBuilder::new().build();
let runtime_api = client.runtime_api();
let best_hash = client.chain_info().best_hash;
assert_eq!(runtime_api.benchmark_add_one(best_hash, &1).unwrap(), 2);
}
#[test]
fn calling_native_runtime_function() {
calling_function_with_strat(ExecutionStrategy::NativeWhenPossible);
}
#[test]
fn calling_wasm_runtime_function() {
calling_function_with_strat(ExecutionStrategy::AlwaysWasm);
}
#[test]
fn calling_native_runtime_signature_changed_function() {
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::NativeWhenPossible)
.build();
let client = TestClientBuilder::new().build();
let runtime_api = client.runtime_api();
let best_hash = client.chain_info().best_hash;
@@ -67,9 +55,7 @@ fn calling_native_runtime_signature_changed_function() {
#[test]
fn use_trie_function() {
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::AlwaysWasm)
.build();
let client = TestClientBuilder::new().build();
let runtime_api = client.runtime_api();
let best_hash = client.chain_info().best_hash;
assert_eq!(runtime_api.use_trie(best_hash).unwrap(), 2);
@@ -77,7 +63,7 @@ fn use_trie_function() {
#[test]
fn initialize_block_works() {
let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build();
let client = TestClientBuilder::new().build();
let runtime_api = client.runtime_api();
let best_hash = client.chain_info().best_hash;
runtime_api
@@ -97,9 +83,7 @@ fn initialize_block_works() {
#[test]
fn record_proof_works() {
let (client, longest_chain) = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::Both)
.build_with_longest_chain();
let (client, longest_chain) = TestClientBuilder::new().build_with_longest_chain();
let storage_root =
*futures::executor::block_on(longest_chain.best_chain()).unwrap().state_root();
@@ -151,7 +135,7 @@ fn record_proof_works() {
#[test]
fn call_runtime_api_with_multiple_arguments() {
let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build();
let client = TestClientBuilder::new().build();
let data = vec![1, 2, 4, 5, 6, 7, 8, 8, 10, 12];
let best_hash = client.chain_info().best_hash;
@@ -166,8 +150,7 @@ fn disable_logging_works() {
if std::env::var("RUN_TEST").is_ok() {
sp_tracing::try_init_simple();
let mut builder =
TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm);
let mut builder = TestClientBuilder::new();
builder.genesis_init_mut().set_wasm_code(
substrate_test_runtime_client::runtime::wasm_binary_logging_disabled_unwrap().to_vec(),
);
@@ -48,42 +48,3 @@ error[E0050]: method `test2` has 2 parameters but the declaration in trait `Api:
| |_^ expected 3 parameters, found 2
|
= note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info)
error[E0050]: method `test_with_context` has 3 parameters but the declaration in trait `Api::test_with_context` has 4
--> tests/ui/mock_only_self_reference.rs:12:1
|
3 | / sp_api::decl_runtime_apis! {
4 | | pub trait Api {
5 | | fn test(data: u64);
| |_________________________- trait requires 4 parameters
...
12 | / sp_api::mock_impl_runtime_apis! {
13 | | impl Api<Block> for MockApi {
14 | | fn test(self, data: u64) {}
15 | |
16 | | fn test2(&mut self, data: u64) {}
17 | | }
18 | | }
| |_^ expected 4 parameters, found 3
|
= note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info)
error[E0050]: method `test2_with_context` has 3 parameters but the declaration in trait `Api::test2_with_context` has 4
--> tests/ui/mock_only_self_reference.rs:12:1
|
3 | / sp_api::decl_runtime_apis! {
4 | | pub trait Api {
5 | | fn test(data: u64);
6 | | fn test2(data: u64);
| |__________________________- trait requires 4 parameters
...
12 | / sp_api::mock_impl_runtime_apis! {
13 | | impl Api<Block> for MockApi {
14 | | fn test(self, data: u64) {}
15 | |
16 | | fn test2(&mut self, data: u64) {}
17 | | }
18 | | }
| |_^ expected 4 parameters, found 3
|
= note: this error originates in the macro `sp_api::mock_impl_runtime_apis` (in Nightly builds, run with -Z macro-backtrace for more info)