Runtime worker threads (#7089)

* std variant

* principal work

* format and naming

* format and naming continued

* working nested fork

* add comment

* naming and tabs

* line width

* fix wording

* address review

* refactor dynamic dispatch

* update wasmtime

* some care

* move ext

* more refactor

* doc effort

* simplify

* doc effort

* tests and docs

* address review

* naming

* explain some args

* add example

* unwinding for native and tests

* rename stray

* fix refs

* fix tests

* fix warnings

* stray naming

* fixes and comments

* Update primitives/io/src/tasks.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* make examples "compile"

* dyn_dispatch -> spawn_call

* fix impl

* address review

* Update primitives/io/src/lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update primitives/io/src/tasks.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update primitives/io/src/async_externalities.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update primitives/io/src/tasks.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/example-parallel/src/lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* fix compilation

* Update client/executor/common/src/wasm_runtime.rs

Co-authored-by: Sergei Shulepov <sergei@parity.io>

* address review

* Update client/executor/wasmtime/src/instance_wrapper.rs

Co-authored-by: Sergei Shulepov <sergei@parity.io>

* Update client/executor/src/native_executor.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update primitives/io/src/tasks.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/executor/src/native_executor.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update primitives/io/src/tasks.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/executor/wasmtime/src/instance_wrapper.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* address some issues

* address more issues

* wasm_only interface

* define sp_tasks

* avoid anyhow

* fix example

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Sergei Shulepov <sergei@parity.io>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/client/executor/Cargo.toml

@@ -18,6 +18,7 @@ derive_more = "0.99.2"
 codec = { package = "parity-scale-codec", version = "1.3.4" }
 sp-io = { version = "2.0.0", path = "../../primitives/io" }
 sp-core = { version = "2.0.0", path = "../../primitives/core" }
+sp-tasks = { version = "2.0.0", path = "../../primitives/tasks" }
 sp-trie = { version = "2.0.0", path = "../../primitives/trie" }
 sp-serializer = { version = "2.0.0", path = "../../primitives/serializer" }
 sp-version = { version = "2.0.0", path = "../../primitives/version" }

substrate/client/executor/common/src/error.rs

@@ -81,6 +81,25 @@ pub enum Error {
 	/// Execution of a host function failed.
 	#[display(fmt="Host function {} execution failed with: {}", _0, _1)]
 	FunctionExecution(String, String),
+	/// No table is present.
+	///
+	/// Call was requested that requires table but none was present in the instance.
+	#[display(fmt="No table exported by wasm blob")]
+	NoTable,
+	/// No table entry is present.
+	///
+	/// Call was requested that requires specific entry in the table to be present.
+	#[display(fmt="No table entry with index {} in wasm blob exported table", _0)]
+	#[from(ignore)]
+	NoTableEntryWithIndex(u32),
+	/// Table entry is not a function.
+	#[display(fmt="Table element with index {} is not a function in wasm blob exported table", _0)]
+	#[from(ignore)]
+	TableElementIsNotAFunction(u32),
+	/// Function in table is null and thus cannot be called.
+	#[display(fmt="Table entry with index {} in wasm blob is null", _0)]
+	#[from(ignore)]
+	FunctionRefIsNull(u32),
 }
 
 impl std::error::Error for Error {

substrate/client/executor/common/src/wasm_runtime.rs

@@ -19,6 +19,46 @@
 use crate::error::Error;
 use sp_wasm_interface::Value;
 
+/// A method to be used to find the entrypoint when calling into the runtime
+///
+/// Contains variants on how to resolve wasm function that will be invoked.
+pub enum InvokeMethod<'a> {
+	/// Call function exported with this name.
+	///
+	/// Located function should have (u32, u32) -> u64 signature.
+	Export(&'a str),
+	/// Call a function found in the exported table found under the given index.
+	///
+	/// Located function should have (u32, u32) -> u64 signature.
+	Table(u32),
+	/// Call function by reference from table through a wrapper.
+	///
+	/// Invoked function (`dispatcher_ref`) function
+	/// should have (u32, u32, u32) -> u64 signature.
+	///
+	/// `func` will be passed to the invoked function as a first argument.
+	TableWithWrapper {
+		/// Wrapper for the call.
+		///
+		/// Function pointer, index into runtime exported table.
+		dispatcher_ref: u32,
+		/// Extra argument for dispatch.
+		///
+		/// Common usage would be to use it as an actual wasm function pointer
+		/// that should be invoked, but can be used as any extra argument on the
+		/// callee side.
+		///
+		/// This is typically generated and invoked by the runtime itself.
+		func: u32,
+	},
+}
+
+impl<'a> From<&'a str> for InvokeMethod<'a> {
+	fn from(val: &'a str) -> InvokeMethod<'a> {
+		InvokeMethod::Export(val)
+	}
+}
+
 /// A trait that defines an abstract WASM runtime module.
 ///
 /// This can be implemented by an execution engine.
@@ -31,11 +71,24 @@ pub trait WasmModule: Sync + Send {
 ///
 /// This can be implemented by an execution engine.
 pub trait WasmInstance: Send {
-	/// Call a method on this WASM instance and reset it afterwards.
+	/// Call a method on this WASM instance.
+	///
+	/// Before execution, instance is reset.
+	///
 	/// Returns the encoded result on success.
-	fn call(&self, method: &str, data: &[u8]) -> Result<Vec<u8>, Error>;
+	fn call(&self, method: InvokeMethod, data: &[u8]) -> Result<Vec<u8>, Error>;
+
+	/// Call an exported method on this WASM instance.
+	///
+	/// Before execution, instance is reset.
+	///
+	/// Returns the encoded result on success.
+	fn call_export(&self, method: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
+		self.call(method.into(), data)
+	}
 
 	/// Get the value from a global with the given `name`.
+	///
 	/// This method is only suitable for getting immutable globals.
 	fn get_global_const(&self, name: &str) -> Result<Option<Value>, Error>;
 }

substrate/client/executor/runtime-test/Cargo.toml

@@ -13,12 +13,13 @@ repository = "https://github.com/paritytech/substrate/"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" }
-sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" }
-sp-sandbox = { version = "0.8.0", default-features = false, path = "../../../primitives/sandbox" }
-sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" }
-sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" }
 sp-allocator = { version = "2.0.0", default-features = false, path = "../../../primitives/allocator" }
+sp-core = { version = "2.0.0", default-features = false, path = "../../../primitives/core" }
+sp-io = { version = "2.0.0", default-features = false, path = "../../../primitives/io" }
+sp-runtime = { version = "2.0.0", default-features = false, path = "../../../primitives/runtime" }
+sp-sandbox = { version = "0.8.0", default-features = false, path = "../../../primitives/sandbox" }
+sp-std = { version = "2.0.0", default-features = false, path = "../../../primitives/std" }
+sp-tasks = { version = "2.0.0", default-features = false, path = "../../../primitives/tasks" }
 
 [build-dependencies]
 wasm-builder-runner = { version = "2.0.0", package = "substrate-wasm-builder-runner", path = "../../../utils/wasm-builder-runner" }
@@ -26,10 +27,11 @@ wasm-builder-runner = { version = "2.0.0", package = "substrate-wasm-builder-run
 [features]
 default = [ "std" ]
 std = [
+	"sp-allocator/std",
+	"sp-core/std",
 	"sp-io/std",
+	"sp-runtime/std",
 	"sp-sandbox/std",
 	"sp-std/std",
-	"sp-core/std",
-	"sp-runtime/std",
-	"sp-allocator/std",
+ 	"sp-tasks/std",
 ]

substrate/client/executor/runtime-test/src/lib.rs

@@ -309,6 +309,43 @@ sp_core::wasm_export_functions! {
 		assert_ne!(test_message, message_slice);
 		message_slice.copy_from_slice(test_message);
 	}
+
+	fn test_spawn() {
+		let data = vec![1u8, 2u8];
+		let data_new = sp_tasks::spawn(tasks::incrementer, data).join();
+
+		assert_eq!(data_new, vec![2u8, 3u8]);
+	}
+
+	fn test_nested_spawn() {
+		let data = vec![7u8, 13u8];
+		let data_new = sp_tasks::spawn(tasks::parallel_incrementer, data).join();
+
+		assert_eq!(data_new, vec![10u8, 16u8]);
+	}
+
+	fn test_panic_in_spawned() {
+		sp_tasks::spawn(tasks::panicker, vec![]).join();
+	}
+ }
+
+ #[cfg(not(feature = "std"))]
+ mod tasks {
+	use sp_std::prelude::*;
+
+	pub fn incrementer(data: Vec<u8>) -> Vec<u8> {
+	   data.into_iter().map(|v| v + 1).collect()
+	}
+
+	pub fn panicker(_: Vec<u8>) -> Vec<u8> {
+		panic!()
+	}
+
+	pub fn parallel_incrementer(data: Vec<u8>) -> Vec<u8> {
+	   let first = data.into_iter().map(|v| v + 2).collect::<Vec<_>>();
+	   let second = sp_tasks::spawn(incrementer, first).join();
+	   second
+	}
  }
 
 #[cfg(not(feature = "std"))]

substrate/client/executor/src/integration_tests/mod.rs

@@ -555,13 +555,13 @@ fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
 	).expect("Creates runtime");
 
 	let instance = runtime.new_instance().unwrap();
-	let res = instance.call("returns_mutable_static", &[0]).unwrap();
+	let res = instance.call_export("returns_mutable_static", &[0]).unwrap();
 	assert_eq!(33, u64::decode(&mut &res[..]).unwrap());
 
 	// We expect that every invocation will need to return the initial
 	// value plus one. If the value increases more than that then it is
 	// a sign that the wasm runtime preserves the memory content.
-	let res = instance.call("returns_mutable_static", &[0]).unwrap();
+	let res = instance.call_export("returns_mutable_static", &[0]).unwrap();
 	assert_eq!(33, u64::decode(&mut &res[..]).unwrap());
 }
 
@@ -590,11 +590,11 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
 	let instance = runtime.new_instance().unwrap();
 
 	// On the first invocation we allocate approx. 768KB (75%) of stack and then trap.
-	let res = instance.call("allocates_huge_stack_array", &true.encode());
+	let res = instance.call_export("allocates_huge_stack_array", &true.encode());
 	assert!(res.is_err());
 
 	// On the second invocation we allocate yet another 768KB (75%) of stack
-	let res = instance.call("allocates_huge_stack_array", &false.encode());
+	let res = instance.call_export("allocates_huge_stack_array", &false.encode());
 	assert!(res.is_ok());
 }
 
@@ -616,10 +616,10 @@ fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) {
 		.expect("`__heap_base` is an `i32`");
 
 	let params = (heap_base as u32, 512u32 * 64 * 1024).encode();
-	instance.call("check_and_set_in_heap", &params).unwrap();
+	instance.call_export("check_and_set_in_heap", &params).unwrap();
 
 	// Cal it a second time to check that the heap was freed.
-	instance.call("check_and_set_in_heap", &params).unwrap();
+	instance.call_export("check_and_set_in_heap", &params).unwrap();
 }
 
 #[test_case(WasmExecutionMethod::Interpreted)]
@@ -720,3 +720,51 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
 	assert_eq!(span_datum.name, "");
 	assert_eq!(values.bool_values.get("wasm").unwrap(), &true);
 }
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn spawning_runtime_instance_should_work(wasm_method: WasmExecutionMethod) {
+
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+
+	call_in_wasm(
+		"test_spawn",
+		&[],
+		wasm_method,
+		&mut ext,
+	).unwrap();
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn spawning_runtime_instance_nested_should_work(wasm_method: WasmExecutionMethod) {
+
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+
+	call_in_wasm(
+		"test_nested_spawn",
+		&[],
+		wasm_method,
+		&mut ext,
+	).unwrap();
+}
+
+#[test_case(WasmExecutionMethod::Interpreted)]
+#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
+fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecutionMethod) {
+
+	let mut ext = TestExternalities::default();
+	let mut ext = ext.ext();
+
+	let error_result = call_in_wasm(
+		"test_panic_in_spawned",
+		&[],
+		wasm_method,
+		&mut ext,
+	).unwrap_err();
+
+	dbg!(&error_result);
+	assert!(format!("{}", error_result).contains("Spawned task"));
+}

substrate/client/executor/src/native_executor.rs

@@ -20,15 +20,28 @@ use crate::{
 	RuntimeInfo, error::{Error, Result},
 	wasm_runtime::{RuntimeCache, WasmExecutionMethod},
 };
+
+use std::{
+	collections::HashMap,
+	panic::{UnwindSafe, AssertUnwindSafe},
+	result,
+	sync::{Arc, atomic::{AtomicU64, Ordering}, mpsc},
+};
+
 use sp_version::{NativeVersion, RuntimeVersion};
 use codec::{Decode, Encode};
 use sp_core::{
-	NativeOrEncoded, traits::{CodeExecutor, Externalities, RuntimeCode, MissingHostFunctions},
+	NativeOrEncoded,
+	traits::{
+		CodeExecutor, Externalities, RuntimeCode, MissingHostFunctions,
+		RuntimeSpawnExt, RuntimeSpawn,
+	},
 };
 use log::trace;
-use std::{result, panic::{UnwindSafe, AssertUnwindSafe}, sync::Arc};
 use sp_wasm_interface::{HostFunctions, Function};
-use sc_executor_common::wasm_runtime::WasmInstance;
+use sc_executor_common::wasm_runtime::{WasmInstance, WasmModule, InvokeMethod};
+use sp_externalities::ExternalitiesExt as _;
+use sp_tasks::new_async_externalities;
 
 /// Default num of pages for the heap
 const DEFAULT_HEAP_PAGES: u64 = 1024;
@@ -136,6 +149,7 @@ impl WasmExecutor {
 		f: F,
 	) -> Result<R>
 		where F: FnOnce(
+			AssertUnwindSafe<&Arc<dyn WasmModule>>,
 			AssertUnwindSafe<&dyn WasmInstance>,
 			Option<&RuntimeVersion>,
 			AssertUnwindSafe<&mut dyn Externalities>,
@@ -148,10 +162,11 @@ impl WasmExecutor {
 			self.default_heap_pages,
 			&*self.host_functions,
 			allow_missing_host_functions,
-			|instance, version, ext| {
+			|module, instance, version, ext| {
+				let module = AssertUnwindSafe(module);
 				let instance = AssertUnwindSafe(instance);
 				let ext = AssertUnwindSafe(ext);
-				f(instance, version, ext)
+				f(module, instance, version, ext)
 			}
 		)? {
 			Ok(r) => r,
@@ -179,10 +194,13 @@ impl sp_core::traits::CallInWasm for WasmExecutor {
 				heap_pages: None,
 			};
 
-			self.with_instance(&code, ext, allow_missing_host_functions, |instance, _, mut ext| {
+			self.with_instance(&code, ext, allow_missing_host_functions, |module, instance, _, mut ext| {
 				with_externalities_safe(
 					&mut **ext,
-					move || instance.call(method, call_data),
+					move || {
+						RuntimeInstanceSpawn::register_on_externalities(module.clone());
+						instance.call_export(method, call_data)
+					}
 				)
 			}).map_err(|e| e.to_string())
 		} else {
@@ -200,10 +218,14 @@ impl sp_core::traits::CallInWasm for WasmExecutor {
 
 			let instance = AssertUnwindSafe(instance);
 			let mut ext = AssertUnwindSafe(ext);
+			let module = AssertUnwindSafe(module);
 
 			with_externalities_safe(
 				&mut **ext,
-				move || instance.call(method, call_data),
+				move || {
+					RuntimeInstanceSpawn::register_on_externalities(module.clone());
+					instance.call_export(method, call_data)
+				}
 			)
 			.and_then(|r| r)
 			.map_err(|e| e.to_string())
@@ -269,12 +291,149 @@ impl<D: NativeExecutionDispatch> RuntimeInfo for NativeExecutor<D> {
 			runtime_code,
 			ext,
 			false,
-			|_instance, version, _ext|
+			|_module, _instance, version, _ext|
 				Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))),
 		)
 	}
 }
 
+/// Helper inner struct to implement `RuntimeSpawn` extension.
+pub struct RuntimeInstanceSpawn {
+	module: Arc<dyn WasmModule>,
+	tasks: parking_lot::Mutex<HashMap<u64, mpsc::Receiver<Vec<u8>>>>,
+	counter: AtomicU64,
+	scheduler: Box<dyn sp_core::traits::SpawnNamed>,
+}
+
+impl RuntimeSpawn for RuntimeInstanceSpawn {
+	fn spawn_call(&self, dispatcher_ref: u32, func: u32, data: Vec<u8>) -> u64 {
+		let new_handle = self.counter.fetch_add(1, Ordering::Relaxed);
+
+		let (sender, receiver) = mpsc::channel();
+		self.tasks.lock().insert(new_handle, receiver);
+
+		let module = self.module.clone();
+		let scheduler = self.scheduler.clone();
+		self.scheduler.spawn("executor-extra-runtime-instance", Box::pin(async move {
+			let module = AssertUnwindSafe(module);
+
+			let async_ext = match new_async_externalities(scheduler.clone()) {
+				Ok(val) => val,
+				Err(e) => {
+					log::error!(
+						target: "executor",
+						"Failed to setup externalities for async context: {}",
+						e,
+					);
+
+					// This will drop sender and receiver end will panic
+					return;
+				}
+			};
+
+			let mut async_ext = match async_ext.with_runtime_spawn(
+				Box::new(RuntimeInstanceSpawn::new(module.clone(), scheduler))
+			) {
+				Ok(val) => val,
+				Err(e) => {
+					log::error!(
+						target: "executor",
+						"Failed to setup runtime extension for async externalities: {}",
+						e,
+					);
+
+					// This will drop sender and receiver end will panic
+					return;
+				}
+			};
+
+			let result = with_externalities_safe(
+				&mut async_ext,
+				move || {
+
+					// FIXME: Should be refactored to shared "instance factory".
+					// Instantiating wasm here every time is suboptimal at the moment, shared
+					// pool of instances should be used.
+					//
+					// https://github.com/paritytech/substrate/issues/7354
+					let instance = module.new_instance()
+						.expect("Failed to create new instance from module");
+
+					instance.call(
+						InvokeMethod::TableWithWrapper { dispatcher_ref, func },
+						&data[..],
+					).expect("Failed to invoke instance.")
+				}
+			);
+
+			match result {
+				Ok(output) => {
+					let _ = sender.send(output);
+				},
+				Err(error) => {
+					// If execution is panicked, the `join` in the original runtime code will panic as well,
+					// since the sender is dropped without sending anything.
+					log::error!("Call error in spawned task: {:?}", error);
+				},
+			}
+		}));
+
+
+		new_handle
+	}
+
+	fn join(&self, handle: u64) -> Vec<u8> {
+		let receiver = self.tasks.lock().remove(&handle).expect("No task for the handle");
+		let output = receiver.recv().expect("Spawned task panicked for the handle");
+		output
+	}
+}
+
+impl RuntimeInstanceSpawn {
+	pub fn new(
+		module: Arc<dyn WasmModule>,
+		scheduler: Box<dyn sp_core::traits::SpawnNamed>,
+	) -> Self {
+		Self {
+			module,
+			scheduler,
+			counter: 0.into(),
+			tasks: HashMap::new().into(),
+		}
+	}
+
+	fn with_externalities_and_module(
+		module: Arc<dyn WasmModule>,
+		mut ext: &mut dyn Externalities,
+	) -> Option<Self> {
+		ext.extension::<sp_core::traits::TaskExecutorExt>()
+			.map(move |task_ext| Self::new(module, task_ext.clone()))
+	}
+
+	/// Register new `RuntimeSpawnExt` on current externalities.
+	///
+	/// This extensions will spawn instances from provided `module`.
+	pub fn register_on_externalities(module: Arc<dyn WasmModule>) {
+		sp_externalities::with_externalities(
+			move |mut ext| {
+				if let Some(runtime_spawn) =
+					Self::with_externalities_and_module(module.clone(), ext)
+				{
+					if let Err(e) = ext.register_extension(
+						RuntimeSpawnExt(Box::new(runtime_spawn))
+					) {
+						trace!(
+							target: "executor",
+							"Failed to register `RuntimeSpawnExt` instance on externalities: {:?}",
+							e,
+						)
+					}
+				}
+			}
+		);
+	}
+}
+
 impl<D: NativeExecutionDispatch + 'static> CodeExecutor for NativeExecutor<D> {
 	type Error = Error;
 
@@ -295,32 +454,34 @@ impl<D: NativeExecutionDispatch + 'static> CodeExecutor for NativeExecutor<D> {
 			runtime_code,
 			ext,
 			false,
-			|instance, onchain_version, mut ext| {
+			|module, instance, onchain_version, mut ext| {
 				let onchain_version = onchain_version.ok_or_else(
 					|| Error::ApiError("Unknown version".into())
 				)?;
+
+				let can_call_with = onchain_version.can_call_with(&self.native_version.runtime_version);
+
 				match (
 					use_native,
-					onchain_version.can_call_with(&self.native_version.runtime_version),
+					can_call_with,
 					native_call,
 				) {
-					(_, false, _) => {
-						trace!(
-							target: "executor",
-							"Request for native execution failed (native: {}, chain: {})",
-							self.native_version.runtime_version,
-							onchain_version,
-						);
+					(_, false, _) | (false, _, _) => {
+						if !can_call_with {
+							trace!(
+								target: "executor",
+								"Request for native execution failed (native: {}, chain: {})",
+								self.native_version.runtime_version,
+								onchain_version,
+							);
+						}
 
 						with_externalities_safe(
 							&mut **ext,
-							move || instance.call(method, data).map(NativeOrEncoded::Encoded)
-						)
-					}
-					(false, _, _) => {
-						with_externalities_safe(
-							&mut **ext,
-							move || instance.call(method, data).map(NativeOrEncoded::Encoded)
+							move || {
+								RuntimeInstanceSpawn::register_on_externalities(module.clone());
+								instance.call_export(method, data).map(NativeOrEncoded::Encoded)
+							}
 						)
 					},
 					(true, true, Some(call)) => {

substrate/client/executor/src/wasm_runtime.rs

@@ -53,7 +53,7 @@ struct VersionedRuntime {
 	/// Wasm runtime type.
 	wasm_method: WasmExecutionMethod,
 	/// Shared runtime that can spawn instances.
-	module: Box<dyn WasmModule>,
+	module: Arc<dyn WasmModule>,
 	/// The number of WebAssembly heap pages this instance was created with.
 	heap_pages: u64,
 	/// Runtime version according to `Core_version` if any.
@@ -70,6 +70,7 @@ impl VersionedRuntime {
 		f: F,
 	) -> Result<R, Error>
 		where F: FnOnce(
+			&Arc<dyn WasmModule>,
 			&dyn WasmInstance,
 			Option<&RuntimeVersion>,
 			&mut dyn Externalities)
@@ -87,7 +88,7 @@ impl VersionedRuntime {
 					.map(|r| Ok((r, false)))
 					.unwrap_or_else(|| self.module.new_instance().map(|i| (i, true)))?;
 
-				let result = f(&*instance, self.version.as_ref(), ext);
+				let result = f(&self.module, &*instance, self.version.as_ref(), ext);
 				if let Err(e) = &result {
 					if new_inst {
 						log::warn!(
@@ -123,7 +124,7 @@ impl VersionedRuntime {
 				// Allocate a new instance
 				let instance = self.module.new_instance()?;
 
-				f(&*instance, self.version.as_ref(), ext)
+				f(&self.module, &*instance, self.version.as_ref(), ext)
 			}
 		}
 	}
@@ -199,6 +200,7 @@ impl RuntimeCache {
 		f: F,
 	) -> Result<Result<R, Error>, Error>
 		where F: FnOnce(
+			&Arc<dyn WasmModule>,
 			&dyn WasmInstance,
 			Option<&RuntimeVersion>,
 			&mut dyn Externalities)
@@ -267,7 +269,7 @@ pub fn create_wasm_runtime_with_code(
 	code: &[u8],
 	host_functions: Vec<&'static dyn Function>,
 	allow_missing_func_imports: bool,
-) -> Result<Box<dyn WasmModule>, WasmError> {
+) -> Result<Arc<dyn WasmModule>, WasmError> {
 	match wasm_method {
 		WasmExecutionMethod::Interpreted =>
 			sc_executor_wasmi::create_runtime(
@@ -275,7 +277,7 @@ pub fn create_wasm_runtime_with_code(
 				heap_pages,
 				host_functions,
 				allow_missing_func_imports
-			).map(|runtime| -> Box<dyn WasmModule> { Box::new(runtime) }),
+			).map(|runtime| -> Arc<dyn WasmModule> { Arc::new(runtime) }),
 		#[cfg(feature = "wasmtime")]
 		WasmExecutionMethod::Compiled =>
 			sc_executor_wasmtime::create_runtime(
@@ -283,7 +285,7 @@ pub fn create_wasm_runtime_with_code(
 				heap_pages,
 				host_functions,
 				allow_missing_func_imports
-			).map(|runtime| -> Box<dyn WasmModule> { Box::new(runtime) }),
+			).map(|runtime| -> Arc<dyn WasmModule> { Arc::new(runtime) }),
 	}
 }
 
@@ -318,7 +320,7 @@ fn create_versioned_wasm_runtime(
 ) -> Result<VersionedRuntime, WasmError> {
 	#[cfg(not(target_os = "unknown"))]
 	let time = std::time::Instant::now();
-	let mut runtime = create_wasm_runtime_with_code(
+	let runtime = create_wasm_runtime_with_code(
 		wasm_method,
 		heap_pages,
 		&code,
@@ -333,10 +335,10 @@ fn create_versioned_wasm_runtime(
 
 		// The following unwind safety assertion is OK because if the method call panics, the
 		// runtime will be dropped.
-		let runtime = AssertUnwindSafe(runtime.as_mut());
+		let runtime = AssertUnwindSafe(runtime.as_ref());
 		crate::native_executor::with_externalities_safe(
 			&mut **ext,
-			move || runtime.new_instance()?.call("Core_version", &[])
+			move || runtime.new_instance()?.call("Core_version".into(), &[])
 		).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))?
 	};
 	let version = match version_result {

substrate/client/executor/wasmi/src/lib.rs

@@ -19,7 +19,7 @@
 use std::{str, cell::RefCell, sync::Arc};
 use wasmi::{
 	Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef,
-	memory_units::Pages,
+	FuncInstance, memory_units::Pages,
 	RuntimeValue::{I32, I64, self},
 };
 use codec::{Encode, Decode};
@@ -29,7 +29,7 @@ use sp_wasm_interface::{
 	FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function,
 };
 use sp_runtime_interface::unpack_ptr_and_len;
-use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance};
+use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance, InvokeMethod};
 use sc_executor_common::{
 	error::{Error, WasmError},
 	sandbox,
@@ -434,7 +434,7 @@ fn get_heap_base(module: &ModuleRef) -> Result<u32, Error> {
 fn call_in_wasm_module(
 	module_instance: &ModuleRef,
 	memory: &MemoryRef,
-	method: &str,
+	method: InvokeMethod,
 	data: &[u8],
 	host_functions: &[&'static dyn Function],
 	allow_missing_func_imports: bool,
@@ -446,24 +446,49 @@ fn call_in_wasm_module(
 		.and_then(|e| e.as_table().cloned());
 	let heap_base = get_heap_base(module_instance)?;
 
-	let mut fec = FunctionExecutor::new(
+	let mut function_executor = FunctionExecutor::new(
 		memory.clone(),
 		heap_base,
-		table,
+		table.clone(),
 		host_functions,
 		allow_missing_func_imports,
 		missing_functions,
 	)?;
 
 	// Write the call data
-	let offset = fec.allocate_memory(data.len() as u32)?;
-	fec.write_memory(offset, data)?;
+	let offset = function_executor.allocate_memory(data.len() as u32)?;
+	function_executor.write_memory(offset, data)?;
 
-	let result = module_instance.invoke_export(
-		method,
-		&[I32(u32::from(offset) as i32), I32(data.len() as i32)],
-		&mut fec,
-	);
+	let result = match method {
+		InvokeMethod::Export(method) => {
+			module_instance.invoke_export(
+				method,
+				&[I32(u32::from(offset) as i32), I32(data.len() as i32)],
+				&mut function_executor,
+			)
+		},
+		InvokeMethod::Table(func_ref) => {
+			let func = table.ok_or(Error::NoTable)?
+				.get(func_ref)?
+				.ok_or(Error::NoTableEntryWithIndex(func_ref))?;
+			FuncInstance::invoke(
+				&func,
+				&[I32(u32::from(offset) as i32), I32(data.len() as i32)],
+				&mut function_executor,
+			).map_err(Into::into)
+		},
+		InvokeMethod::TableWithWrapper { dispatcher_ref, func } => {
+			let dispatcher = table.ok_or(Error::NoTable)?
+				.get(dispatcher_ref)?
+				.ok_or(Error::NoTableEntryWithIndex(dispatcher_ref))?;
+
+			FuncInstance::invoke(
+				&dispatcher,
+				&[I32(func as _), I32(u32::from(offset) as i32), I32(data.len() as i32)],
+				&mut function_executor,
+			).map_err(Into::into)
+		},
+	};
 
 	match result {
 		Ok(Some(I64(r))) => {
@@ -474,7 +499,7 @@ fn call_in_wasm_module(
 			trace!(
 				target: "wasm-executor",
 				"Failed to execute code with {} pages",
-				memory.current_size().0
+				memory.current_size().0,
 			);
 			Err(e.into())
 		},
@@ -677,7 +702,7 @@ pub struct WasmiInstance {
 unsafe impl Send for WasmiInstance {}
 
 impl WasmInstance for WasmiInstance {
-	fn call(&self, method: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
+	fn call(&self, method: InvokeMethod, data: &[u8]) -> Result<Vec<u8>, Error> {
 		// We reuse a single wasm instance for multiple calls and a previous call (if any)
 		// altered the state. Therefore, we need to restore the instance to original state.
 

substrate/client/executor/wasmtime/src/instance_wrapper.rs

@@ -26,6 +26,7 @@ use std::{slice, marker};
 use sc_executor_common::{
 	error::{Error, Result},
 	util::{WasmModuleInfo, DataSegmentsSnapshot},
+	wasm_runtime::InvokeMethod,
 };
 use sp_wasm_interface::{Pointer, WordSize, Value};
 use wasmtime::{Engine, Instance, Module, Memory, Table, Val, Func, Extern, Global, Store};
@@ -72,6 +73,82 @@ impl ModuleWrapper {
 	}
 }
 
+/// Invoked entrypoint format.
+pub enum EntryPointType {
+	/// Direct call.
+	///
+	/// Call is made by providing only payload reference and length.
+	Direct,
+	/// Indirect call.
+	///
+	/// Call is made by providing payload reference and length, and extra argument
+	/// for advanced routing (typically extra WASM function pointer).
+	Wrapped(u32),
+}
+
+/// Wasm blob entry point.
+pub struct EntryPoint {
+	call_type: EntryPointType,
+	func: wasmtime::Func,
+}
+
+impl EntryPoint {
+	/// Call this entry point.
+	pub fn call(&self, data_ptr: Pointer<u8>, data_len: WordSize) -> Result<u64> {
+		let data_ptr = u32::from(data_ptr) as i32;
+		let data_len = u32::from(data_len) as i32;
+
+		(match self.call_type {
+			EntryPointType::Direct => {
+				self.func.call(&[
+					wasmtime::Val::I32(data_ptr),
+					wasmtime::Val::I32(data_len),
+				])
+			},
+			EntryPointType::Wrapped(func) => {
+				self.func.call(&[
+					wasmtime::Val::I32(func as _),
+					wasmtime::Val::I32(data_ptr),
+					wasmtime::Val::I32(data_len),
+				])
+			},
+		})
+			.map(|results| 
+				// the signature is checked to have i64 return type
+				results[0].unwrap_i64() as u64
+			)
+			.map_err(|err| Error::from(format!(
+				"Wasm execution trapped: {}",
+				err
+			)))
+	}
+
+	pub fn direct(func: wasmtime::Func) -> std::result::Result<Self, &'static str> {
+		match (func.ty().params(), func.ty().results()) {
+			(&[wasmtime::ValType::I32, wasmtime::ValType::I32], &[wasmtime::ValType::I64]) => {
+				Ok(Self { func, call_type: EntryPointType::Direct })
+			}
+			_ => {
+				Err("Invalid signature for direct entry point")
+			}
+		}
+	}
+
+	pub fn wrapped(dispatcher: wasmtime::Func, func: u32) -> std::result::Result<Self, &'static str> {
+		match (dispatcher.ty().params(), dispatcher.ty().results()) {
+			(
+				&[wasmtime::ValType::I32, wasmtime::ValType::I32, wasmtime::ValType::I32],
+				&[wasmtime::ValType::I64],
+			) => {
+				Ok(Self { func: dispatcher, call_type: EntryPointType::Wrapped(func) })
+			},
+			_ => {
+				Err("Invalid signature for wrapped entry point")
+			}
+		}
+	}
+}
+
 /// Wrap the given WebAssembly Instance of a wasm module with Substrate-runtime.
 ///
 /// This struct is a handy wrapper around a wasmtime `Instance` that provides substrate specific
@@ -150,24 +227,62 @@ impl InstanceWrapper {
 	///
 	/// An entrypoint must have a signature `(i32, i32) -> i64`, otherwise this function will return
 	/// an error.
-	pub fn resolve_entrypoint(&self, name: &str) -> Result<wasmtime::Func> {
-		// Resolve the requested method and verify that it has a proper signature.
-		let export = self
-			.instance
-			.get_export(name)
-			.ok_or_else(|| Error::from(format!("Exported method {} is not found", name)))?;
-		let entrypoint = extern_func(&export)
-			.ok_or_else(|| Error::from(format!("Export {} is not a function", name)))?;
-		match (entrypoint.ty().params(), entrypoint.ty().results()) {
-			(&[wasmtime::ValType::I32, wasmtime::ValType::I32], &[wasmtime::ValType::I64]) => {}
-			_ => {
-				return Err(Error::from(format!(
-					"method {} have an unsupported signature",
-					name
-				)))
-			}
-		}
-		Ok(entrypoint.clone())
+	pub fn resolve_entrypoint(&self, method: InvokeMethod) -> Result<EntryPoint> {
+		Ok(match method {
+			InvokeMethod::Export(method) => {
+				// Resolve the requested method and verify that it has a proper signature.
+				let export = self
+					.instance
+					.get_export(method)
+					.ok_or_else(|| Error::from(format!("Exported method {} is not found", method)))?;
+				let func = extern_func(&export)
+					.ok_or_else(|| Error::from(format!("Export {} is not a function", method)))?
+					.clone();
+				EntryPoint::direct(func)
+					.map_err(|_|
+						Error::from(format!(
+							"Exported function '{}' has invalid signature.",
+							method,
+						))
+					)?
+			},
+			InvokeMethod::Table(func_ref) => {
+				let table = self.instance.get_table("__indirect_function_table").ok_or(Error::NoTable)?;
+				let val = table.get(func_ref)
+					.ok_or(Error::NoTableEntryWithIndex(func_ref))?;
+				let func = val
+					.funcref()
+					.ok_or(Error::TableElementIsNotAFunction(func_ref))?
+					.ok_or(Error::FunctionRefIsNull(func_ref))?
+					.clone();
+
+				EntryPoint::direct(func)
+					.map_err(|_|
+						Error::from(format!(
+							"Function @{} in exported table has invalid signature for direct call.",
+							func_ref,
+						))
+					)?
+				},
+			InvokeMethod::TableWithWrapper { dispatcher_ref, func } => {
+				let table = self.instance.get_table("__indirect_function_table").ok_or(Error::NoTable)?;
+				let val = table.get(dispatcher_ref)
+					.ok_or(Error::NoTableEntryWithIndex(dispatcher_ref))?;
+				let dispatcher = val
+					.funcref()
+					.ok_or(Error::TableElementIsNotAFunction(dispatcher_ref))?
+					.ok_or(Error::FunctionRefIsNull(dispatcher_ref))?
+					.clone();
+
+				EntryPoint::wrapped(dispatcher, func)
+					.map_err(|_|
+						Error::from(format!(
+							"Function @{} in exported table has invalid signature for wrapped call.",
+							dispatcher_ref,
+						))
+					)?
+			},
+		})
 	}
 
 	/// Returns an indirect function table of this instance.

substrate/client/executor/wasmtime/src/runtime.rs

@@ -18,14 +18,14 @@
 
 use crate::host::HostState;
 use crate::imports::{Imports, resolve_imports};
-use crate::instance_wrapper::{ModuleWrapper, InstanceWrapper, GlobalsSnapshot};
+use crate::instance_wrapper::{ModuleWrapper, InstanceWrapper, GlobalsSnapshot, EntryPoint};
 use crate::state_holder;
 
 use std::rc::Rc;
 use std::sync::Arc;
 use sc_executor_common::{
-	error::{Error, Result, WasmError},
-	wasm_runtime::{WasmModule, WasmInstance},
+	error::{Result, WasmError},
+	wasm_runtime::{WasmModule, WasmInstance, InvokeMethod},
 };
 use sp_allocator::FreeingBumpHeapAllocator;
 use sp_runtime_interface::unpack_ptr_and_len;
@@ -90,7 +90,7 @@ pub struct WasmtimeInstance {
 unsafe impl Send for WasmtimeInstance {}
 
 impl WasmInstance for WasmtimeInstance {
-	fn call(&self, method: &str, data: &[u8]) -> Result<Vec<u8>> {
+	fn call(&self, method: InvokeMethod, data: &[u8]) -> Result<Vec<u8>> {
 		let entrypoint = self.instance_wrapper.resolve_entrypoint(method)?;
 		let allocator = FreeingBumpHeapAllocator::new(self.heap_base);
 
@@ -146,28 +146,14 @@ pub fn create_runtime(
 fn perform_call(
 	data: &[u8],
 	instance_wrapper: Rc<InstanceWrapper>,
-	entrypoint: wasmtime::Func,
+	entrypoint: EntryPoint,
 	mut allocator: FreeingBumpHeapAllocator,
 ) -> Result<Vec<u8>> {
 	let (data_ptr, data_len) = inject_input_data(&instance_wrapper, &mut allocator, data)?;
 
 	let host_state = HostState::new(allocator, instance_wrapper.clone());
-	let ret = state_holder::with_initialized_state(&host_state, || {
-		match entrypoint.call(&[
-			wasmtime::Val::I32(u32::from(data_ptr) as i32),
-			wasmtime::Val::I32(u32::from(data_len) as i32),
-		]) {
-			Ok(results) => {
-				let retval = results[0].unwrap_i64() as u64;
-				Ok(unpack_ptr_and_len(retval))
-			}
-			Err(trap) => {
-				return Err(Error::from(format!(
-					"Wasm execution trapped: {}",
-					trap
-				)));
-			}
-		}
+	let ret = state_holder::with_initialized_state(&host_state, || -> Result<_> {
+		Ok(unpack_ptr_and_len(entrypoint.call(data_ptr, data_len)?))
 	});
 	let (output_ptr, output_len) = ret?;
 	let output = extract_output_data(&instance_wrapper, output_ptr, output_len)?;