Statically register host WASM functions (#10394)

* Statically register host WASM functions

* Fix `substrate-test-client` compilation

* Move `ExtendedHostFunctions` to `sp-wasm-interface`

* Fix `sp-runtime-interface` tests' compilation

* Fix `sc-executor-wasmtime` tests' compilation

* Use `runtime_interface` macro in `test-runner`

* Fix `sc-executor` tests' compilation

* Reformatting/`rustfmt`

* Add an extra comment regarding the `H` generic arg in `create_runtime`

* Even more `rustfmt`

* Depend on `wasmtime` without default features in `sp-wasm-interface`

* Bump version of `sp-wasm-interface` to 4.0.1

* Bump `sp-wasm-interface` in `Cargo.lock` too

* Bump all of the `sp-wasm-interface` requirements to 4.0.1

Maybe this will appease cargo-unleash?

* Revert "Bump all of the `sp-wasm-interface` requirements to 4.0.1"

This reverts commit 0f7ccf8e0f371542861121b145ab87af6541ac30.

* Make `cargo-unleash` happy (maybe)

* Use `cargo-unleash` to bump the crates' versions

* Align to review comments

substrate/client/executor/wasmtime/src/host.rs

@@ -62,11 +62,11 @@ impl HostState {
 /// A `HostContext` implements `FunctionContext` for making host calls from a Wasmtime
 /// runtime. The `HostContext` exists only for the lifetime of the call and borrows state from
 /// a longer-living `HostState`.
-pub(crate) struct HostContext<'a, 'b> {
-	pub(crate) caller: &'a mut Caller<'b, StoreData>,
+pub(crate) struct HostContext<'a> {
+	pub(crate) caller: Caller<'a, StoreData>,
 }
 
-impl<'a, 'b> HostContext<'a, 'b> {
+impl<'a> HostContext<'a> {
 	fn host_state(&self) -> &HostState {
 		self.caller
 			.data()
@@ -98,7 +98,7 @@ impl<'a, 'b> HostContext<'a, 'b> {
 	}
 }
 
-impl<'a, 'b> sp_wasm_interface::FunctionContext for HostContext<'a, 'b> {
+impl<'a> sp_wasm_interface::FunctionContext for HostContext<'a> {
 	fn read_memory_into(
 		&self,
 		address: Pointer<u8>,
@@ -136,7 +136,7 @@ impl<'a, 'b> sp_wasm_interface::FunctionContext for HostContext<'a, 'b> {
 	}
 }
 
-impl<'a, 'b> Sandbox for HostContext<'a, 'b> {
+impl<'a> Sandbox for HostContext<'a> {
 	fn memory_get(
 		&mut self,
 		memory_id: MemoryId,
@@ -320,12 +320,12 @@ impl<'a, 'b> Sandbox for HostContext<'a, 'b> {
 	}
 }
 
-struct SandboxContext<'a, 'b, 'c> {
-	host_context: &'a mut HostContext<'b, 'c>,
+struct SandboxContext<'a, 'b> {
+	host_context: &'a mut HostContext<'b>,
 	dispatch_thunk: Func,
 }
 
-impl<'a, 'b, 'c> sandbox::SandboxContext for SandboxContext<'a, 'b, 'c> {
+impl<'a, 'b> sandbox::SandboxContext for SandboxContext<'a, 'b> {
 	fn invoke(
 		&mut self,
 		invoke_args_ptr: Pointer<u8>,

substrate/client/executor/wasmtime/src/imports.rs

@@ -19,14 +19,11 @@
 use crate::{
 	host::HostContext,
 	runtime::{Store, StoreData},
-	util,
 };
 use sc_executor_common::error::WasmError;
-use sp_wasm_interface::{Function, ValueType};
-use std::{any::Any, convert::TryInto};
-use wasmtime::{
-	Caller, Extern, ExternType, Func, FuncType, ImportType, Memory, MemoryType, Module, Trap, Val,
-};
+use sp_wasm_interface::{FunctionContext, HostFunctions};
+use std::{collections::HashMap, convert::TryInto};
+use wasmtime::{Extern, ExternType, Func, FuncType, ImportType, Memory, MemoryType, Module, Trap};
 
 pub struct Imports {
 	/// Contains the index into `externs` where the memory import is stored if any. `None` if there
@@ -37,16 +34,19 @@ pub struct Imports {
 
 /// Goes over all imports of a module and prepares a vector of `Extern`s that can be used for
 /// instantiation of the module. Returns an error if there are imports that cannot be satisfied.
-pub(crate) fn resolve_imports(
+pub(crate) fn resolve_imports<H>(
 	store: &mut Store,
 	module: &Module,
-	host_functions: &[&'static dyn Function],
 	heap_pages: u64,
 	allow_missing_func_imports: bool,
-) -> Result<Imports, WasmError> {
+) -> Result<Imports, WasmError>
+where
+	H: HostFunctions,
+{
 	let mut externs = vec![];
 	let mut memory_import_index = None;
-	for import_ty in module.imports() {
+	let mut pending_func_imports = HashMap::new();
+	for (index, import_ty) in module.imports().enumerate() {
 		let name = import_name(&import_ty)?;
 
 		if import_ty.module() != "env" {
@@ -57,19 +57,89 @@ pub(crate) fn resolve_imports(
 			)))
 		}
 
-		let resolved = match name {
-			"memory" => {
-				memory_import_index = Some(externs.len());
-				resolve_memory_import(store, &import_ty, heap_pages)?
+		if name == "memory" {
+			memory_import_index = Some(index);
+			externs.push((index, resolve_memory_import(store, &import_ty, heap_pages)?));
+			continue
+		}
+
+		match import_ty.ty() {
+			ExternType::Func(func_ty) => {
+				pending_func_imports.insert(name.to_owned(), (index, import_ty, func_ty));
 			},
 			_ =>
-				resolve_func_import(store, &import_ty, host_functions, allow_missing_func_imports)?,
+				return Err(WasmError::Other(format!(
+					"host doesn't provide any non function imports besides 'memory': {}:{}",
+					import_ty.module(),
+					name,
+				))),
 		};
-		externs.push(resolved);
 	}
+
+	let mut registry = Registry { store, externs, pending_func_imports };
+
+	H::register_static(&mut registry)?;
+	let mut externs = registry.externs;
+
+	if !registry.pending_func_imports.is_empty() {
+		if allow_missing_func_imports {
+			for (_, (index, import_ty, func_ty)) in registry.pending_func_imports {
+				externs.push((
+					index,
+					MissingHostFuncHandler::new(&import_ty)?.into_extern(store, &func_ty),
+				));
+			}
+		} else {
+			let mut names = Vec::new();
+			for (name, (_, import_ty, _)) in registry.pending_func_imports {
+				names.push(format!("'{}:{}'", import_ty.module(), name));
+			}
+			let names = names.join(", ");
+			return Err(WasmError::Other(format!(
+				"runtime requires function imports which are not present on the host: {}",
+				names
+			)))
+		}
+	}
+
+	externs.sort_unstable_by_key(|&(index, _)| index);
+	let externs = externs.into_iter().map(|(_, ext)| ext).collect();
+
 	Ok(Imports { memory_import_index, externs })
 }
 
+struct Registry<'a, 'b> {
+	store: &'a mut Store,
+	externs: Vec<(usize, Extern)>,
+	pending_func_imports: HashMap<String, (usize, ImportType<'b>, FuncType)>,
+}
+
+impl<'a, 'b> sp_wasm_interface::HostFunctionRegistry for Registry<'a, 'b> {
+	type State = StoreData;
+	type Error = WasmError;
+	type FunctionContext = HostContext<'a>;
+
+	fn with_function_context<R>(
+		caller: wasmtime::Caller<Self::State>,
+		callback: impl FnOnce(&mut dyn FunctionContext) -> R,
+	) -> R {
+		callback(&mut HostContext { caller })
+	}
+
+	fn register_static<Params, Results>(
+		&mut self,
+		fn_name: &str,
+		func: impl wasmtime::IntoFunc<Self::State, Params, Results>,
+	) -> Result<(), Self::Error> {
+		if let Some((index, _, _)) = self.pending_func_imports.remove(fn_name) {
+			let func = Func::wrap(&mut *self.store, func);
+			self.externs.push((index, Extern::Func(func)));
+		}
+
+		Ok(())
+	}
+}
+
 /// When the module linking proposal is supported the import's name can be `None`.
 /// Because we are not using this proposal we could safely unwrap the name.
 /// However, we opt for an error in order to avoid panics at all costs.
@@ -139,115 +209,6 @@ fn resolve_memory_import(
 	Ok(Extern::Memory(memory))
 }
 
-fn resolve_func_import(
-	store: &mut Store,
-	import_ty: &ImportType,
-	host_functions: &[&'static dyn Function],
-	allow_missing_func_imports: bool,
-) -> Result<Extern, WasmError> {
-	let name = import_name(&import_ty)?;
-
-	let func_ty = match import_ty.ty() {
-		ExternType::Func(func_ty) => func_ty,
-		_ =>
-			return Err(WasmError::Other(format!(
-				"host doesn't provide any non function imports besides 'memory': {}:{}",
-				import_ty.module(),
-				name,
-			))),
-	};
-
-	let host_func = match host_functions.iter().find(|host_func| host_func.name() == name) {
-		Some(host_func) => host_func,
-		None if allow_missing_func_imports =>
-			return Ok(MissingHostFuncHandler::new(import_ty)?.into_extern(store, &func_ty)),
-		None =>
-			return Err(WasmError::Other(format!(
-				"host doesn't provide such function: {}:{}",
-				import_ty.module(),
-				name,
-			))),
-	};
-	if &func_ty != &wasmtime_func_sig(*host_func) {
-		return Err(WasmError::Other(format!(
-			"signature mismatch for: {}:{}",
-			import_ty.module(),
-			name,
-		)))
-	}
-
-	Ok(HostFuncHandler::new(*host_func).into_extern(store))
-}
-
-/// This structure implements `Callable` and acts as a bridge between wasmtime and
-/// substrate host functions.
-struct HostFuncHandler {
-	host_func: &'static dyn Function,
-}
-
-fn call_static<'a>(
-	static_func: &'static dyn Function,
-	wasmtime_params: &[Val],
-	wasmtime_results: &mut [Val],
-	mut caller: Caller<'a, StoreData>,
-) -> Result<(), wasmtime::Trap> {
-	let unwind_result = {
-		let mut host_ctx = HostContext { caller: &mut caller };
-
-		// `from_wasmtime_val` panics if it encounters a value that doesn't fit into the values
-		// available in substrate.
-		//
-		// This, however, cannot happen since the signature of this function is created from
-		// a `dyn Function` signature of which cannot have a non substrate value by definition.
-		let mut params = wasmtime_params.iter().cloned().map(util::from_wasmtime_val);
-
-		std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
-			static_func.execute(&mut host_ctx, &mut params)
-		}))
-	};
-
-	let execution_result = match unwind_result {
-		Ok(execution_result) => execution_result,
-		Err(err) => return Err(Trap::new(stringify_panic_payload(err))),
-	};
-
-	match execution_result {
-		Ok(Some(ret_val)) => {
-			debug_assert!(
-				wasmtime_results.len() == 1,
-				"wasmtime function signature, therefore the number of results, should always \
-				correspond to the number of results returned by the host function",
-			);
-			wasmtime_results[0] = util::into_wasmtime_val(ret_val);
-			Ok(())
-		},
-		Ok(None) => {
-			debug_assert!(
-				wasmtime_results.len() == 0,
-				"wasmtime function signature, therefore the number of results, should always \
-				correspond to the number of results returned by the host function",
-			);
-			Ok(())
-		},
-		Err(msg) => Err(Trap::new(msg)),
-	}
-}
-
-impl HostFuncHandler {
-	fn new(host_func: &'static dyn Function) -> Self {
-		Self { host_func }
-	}
-
-	fn into_extern(self, store: &mut Store) -> Extern {
-		let host_func = self.host_func;
-		let func_ty = wasmtime_func_sig(self.host_func);
-		let func = Func::new(store, func_ty, move |caller, params, result| {
-			call_static(host_func, params, result, caller)
-		});
-		Extern::Func(func)
-	}
-}
-
 /// A `Callable` handler for missing functions.
 struct MissingHostFuncHandler {
 	module: String,
@@ -270,33 +231,3 @@ impl MissingHostFuncHandler {
 		Extern::Func(func)
 	}
 }
-
-fn wasmtime_func_sig(func: &dyn Function) -> wasmtime::FuncType {
-	let signature = func.signature();
-	let params = signature.args.iter().cloned().map(into_wasmtime_val_type);
-
-	let results = signature.return_value.iter().cloned().map(into_wasmtime_val_type);
-
-	wasmtime::FuncType::new(params, results)
-}
-
-fn into_wasmtime_val_type(val_ty: ValueType) -> wasmtime::ValType {
-	match val_ty {
-		ValueType::I32 => wasmtime::ValType::I32,
-		ValueType::I64 => wasmtime::ValType::I64,
-		ValueType::F32 => wasmtime::ValType::F32,
-		ValueType::F64 => wasmtime::ValType::F64,
-	}
-}
-
-/// Attempt to convert a opaque panic payload to a string.
-fn stringify_panic_payload(payload: Box<dyn Any + Send + 'static>) -> String {
-	match payload.downcast::<&'static str>() {
-		Ok(msg) => msg.to_string(),
-		Err(payload) => match payload.downcast::<String>() {
-			Ok(msg) => *msg,
-			// At least we tried...
-			Err(_) => "Box<Any>".to_string(),
-		},
-	}
-}

substrate/client/executor/wasmtime/src/instance_wrapper.rs

@@ -24,7 +24,7 @@ use sc_executor_common::{
 	error::{Error, Result},
 	wasm_runtime::InvokeMethod,
 };
-use sp_wasm_interface::{Function, Pointer, Value, WordSize};
+use sp_wasm_interface::{HostFunctions, Pointer, Value, WordSize};
 use wasmtime::{
 	AsContext, AsContextMut, Extern, Func, Global, Instance, Memory, Module, Table, Val,
 };
@@ -138,13 +138,15 @@ fn extern_func(extern_: &Extern) -> Option<&Func> {
 
 impl InstanceWrapper {
 	/// Create a new instance wrapper from the given wasm module.
-	pub fn new(
+	pub fn new<H>(
 		module: &Module,
-		host_functions: &[&'static dyn Function],
 		heap_pages: u64,
 		allow_missing_func_imports: bool,
 		max_memory_size: Option<usize>,
-	) -> Result<Self> {
+	) -> Result<Self>
+	where
+		H: HostFunctions,
+	{
 		let limits = if let Some(max_memory_size) = max_memory_size {
 			wasmtime::StoreLimitsBuilder::new().memory_size(max_memory_size).build()
 		} else {
@@ -161,10 +163,9 @@ impl InstanceWrapper {
 
 		// Scan all imports, find the matching host functions, and create stubs that adapt arguments
 		// and results.
-		let imports = crate::imports::resolve_imports(
+		let imports = crate::imports::resolve_imports::<H>(
 			&mut store,
 			module,
-			host_functions,
 			heap_pages,
 			allow_missing_func_imports,
 		)?;

substrate/client/executor/wasmtime/src/runtime.rs

@@ -23,6 +23,7 @@ use crate::{
 	instance_wrapper::{EntryPoint, InstanceWrapper},
 	util,
 };
+use core::marker::PhantomData;
 
 use sc_allocator::FreeingBumpHeapAllocator;
 use sc_executor_common::{
@@ -33,7 +34,7 @@ use sc_executor_common::{
 	wasm_runtime::{InvokeMethod, WasmInstance, WasmModule},
 };
 use sp_runtime_interface::unpack_ptr_and_len;
-use sp_wasm_interface::{Function, Pointer, Value, WordSize};
+use sp_wasm_interface::{HostFunctions, Pointer, Value, WordSize};
 use std::{
 	path::{Path, PathBuf},
 	sync::{
@@ -79,29 +80,31 @@ impl StoreData {
 
 pub(crate) type Store = wasmtime::Store<StoreData>;
 
-enum Strategy {
+enum Strategy<H> {
 	FastInstanceReuse {
 		instance_wrapper: InstanceWrapper,
 		globals_snapshot: GlobalsSnapshot<wasmtime::Global>,
 		data_segments_snapshot: Arc<DataSegmentsSnapshot>,
 		heap_base: u32,
 	},
-	RecreateInstance(InstanceCreator),
+	RecreateInstance(InstanceCreator<H>),
 }
 
-struct InstanceCreator {
+struct InstanceCreator<H> {
 	module: Arc<wasmtime::Module>,
-	host_functions: Vec<&'static dyn Function>,
 	heap_pages: u64,
 	allow_missing_func_imports: bool,
 	max_memory_size: Option<usize>,
+	phantom: PhantomData<H>,
 }
 
-impl InstanceCreator {
+impl<H> InstanceCreator<H>
+where
+	H: HostFunctions,
+{
 	fn instantiate(&mut self) -> Result<InstanceWrapper> {
-		InstanceWrapper::new(
+		InstanceWrapper::new::<H>(
 			&*self.module,
-			&self.host_functions,
 			self.heap_pages,
 			self.allow_missing_func_imports,
 			self.max_memory_size,
@@ -141,19 +144,21 @@ struct InstanceSnapshotData {
 
 /// A `WasmModule` implementation using wasmtime to compile the runtime module to machine code
 /// and execute the compiled code.
-pub struct WasmtimeRuntime {
+pub struct WasmtimeRuntime<H> {
 	module: Arc<wasmtime::Module>,
 	snapshot_data: Option<InstanceSnapshotData>,
 	config: Config,
-	host_functions: Vec<&'static dyn Function>,
+	phantom: PhantomData<H>,
 }
 
-impl WasmModule for WasmtimeRuntime {
+impl<H> WasmModule for WasmtimeRuntime<H>
+where
+	H: HostFunctions,
+{
 	fn new_instance(&self) -> Result<Box<dyn WasmInstance>> {
 		let strategy = if let Some(ref snapshot_data) = self.snapshot_data {
-			let mut instance_wrapper = InstanceWrapper::new(
+			let mut instance_wrapper = InstanceWrapper::new::<H>(
 				&self.module,
-				&self.host_functions,
 				self.config.heap_pages,
 				self.config.allow_missing_func_imports,
 				self.config.max_memory_size,
@@ -169,19 +174,19 @@ impl WasmModule for WasmtimeRuntime {
 				&mut InstanceGlobals { instance: &mut instance_wrapper },
 			);
 
-			Strategy::FastInstanceReuse {
+			Strategy::<H>::FastInstanceReuse {
 				instance_wrapper,
 				globals_snapshot,
 				data_segments_snapshot: snapshot_data.data_segments_snapshot.clone(),
 				heap_base,
 			}
 		} else {
-			Strategy::RecreateInstance(InstanceCreator {
+			Strategy::<H>::RecreateInstance(InstanceCreator {
 				module: self.module.clone(),
-				host_functions: self.host_functions.clone(),
 				heap_pages: self.config.heap_pages,
 				allow_missing_func_imports: self.config.allow_missing_func_imports,
 				max_memory_size: self.config.max_memory_size,
+				phantom: PhantomData,
 			})
 		};
 
@@ -191,11 +196,14 @@ impl WasmModule for WasmtimeRuntime {
 
 /// A `WasmInstance` implementation that reuses compiled module and spawns instances
 /// to execute the compiled code.
-pub struct WasmtimeInstance {
-	strategy: Strategy,
+pub struct WasmtimeInstance<H> {
+	strategy: Strategy<H>,
 }
 
-impl WasmInstance for WasmtimeInstance {
+impl<H> WasmInstance for WasmtimeInstance<H>
+where
+	H: HostFunctions,
+{
 	fn call(&mut self, method: InvokeMethod, data: &[u8]) -> Result<Vec<u8>> {
 		match &mut self.strategy {
 			Strategy::FastInstanceReuse {
@@ -483,13 +491,18 @@ enum CodeSupplyMode<'a> {
 
 /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to
 /// machine code, which can be computationally heavy.
-pub fn create_runtime(
+///
+/// The `H` generic parameter is used to statically pass a set of host functions which are exposed
+/// to the runtime.
+pub fn create_runtime<H>(
 	blob: RuntimeBlob,
 	config: Config,
-	host_functions: Vec<&'static dyn Function>,
-) -> std::result::Result<WasmtimeRuntime, WasmError> {
+) -> std::result::Result<WasmtimeRuntime<H>, WasmError>
+where
+	H: HostFunctions,
+{
 	// SAFETY: this is safe because it doesn't use `CodeSupplyMode::Artifact`.
-	unsafe { do_create_runtime(CodeSupplyMode::Verbatim { blob }, config, host_functions) }
+	unsafe { do_create_runtime::<H>(CodeSupplyMode::Verbatim { blob }, config) }
 }
 
 /// The same as [`create_runtime`] but takes a precompiled artifact, which makes this function
@@ -503,23 +516,27 @@ pub fn create_runtime(
 ///
 /// It is ok though if the `compiled_artifact` was created by code of another version or with
 /// different configuration flags. In such case the caller will receive an `Err` deterministically.
-pub unsafe fn create_runtime_from_artifact(
+pub unsafe fn create_runtime_from_artifact<H>(
 	compiled_artifact: &[u8],
 	config: Config,
-	host_functions: Vec<&'static dyn Function>,
-) -> std::result::Result<WasmtimeRuntime, WasmError> {
-	do_create_runtime(CodeSupplyMode::Artifact { compiled_artifact }, config, host_functions)
+) -> std::result::Result<WasmtimeRuntime<H>, WasmError>
+where
+	H: HostFunctions,
+{
+	do_create_runtime::<H>(CodeSupplyMode::Artifact { compiled_artifact }, config)
 }
 
 /// # Safety
 ///
 /// This is only unsafe if called with [`CodeSupplyMode::Artifact`]. See
 /// [`create_runtime_from_artifact`] to get more details.
-unsafe fn do_create_runtime(
+unsafe fn do_create_runtime<H>(
 	code_supply_mode: CodeSupplyMode<'_>,
 	config: Config,
-	host_functions: Vec<&'static dyn Function>,
-) -> std::result::Result<WasmtimeRuntime, WasmError> {
+) -> std::result::Result<WasmtimeRuntime<H>, WasmError>
+where
+	H: HostFunctions,
+{
 	// Create the engine, store and finally the module from the given code.
 	let mut wasmtime_config = common_config(&config.semantics)?;
 	if let Some(ref cache_path) = config.cache_path {
@@ -566,7 +583,7 @@ unsafe fn do_create_runtime(
 		},
 	};
 
-	Ok(WasmtimeRuntime { module: Arc::new(module), snapshot_data, config, host_functions })
+	Ok(WasmtimeRuntime { module: Arc::new(module), snapshot_data, config, phantom: PhantomData })
 }
 
 fn instrument(

substrate/client/executor/wasmtime/src/tests.rs

@@ -78,7 +78,7 @@ impl RuntimeBuilder {
 				.expect("failed to create a runtime blob out of test runtime")
 		};
 
-		let rt = crate::create_runtime(
+		let rt = crate::create_runtime::<HostFunctions>(
 			blob,
 			crate::Config {
 				heap_pages: self.heap_pages,
@@ -98,10 +98,6 @@ impl RuntimeBuilder {
 					parallel_compilation: true,
 				},
 			},
-			{
-				use sp_wasm_interface::HostFunctions as _;
-				HostFunctions::host_functions()
-			},
 		)
 		.expect("cannot create runtime");
 
@@ -316,9 +312,7 @@ fn test_max_memory_pages() {
 #[cfg_attr(build_type = "debug", ignore)]
 #[test]
 fn test_instances_without_reuse_are_not_leaked() {
-	use sp_wasm_interface::HostFunctions;
-
-	let runtime = crate::create_runtime(
+	let runtime = crate::create_runtime::<HostFunctions>(
 		RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
 		crate::Config {
 			heap_pages: 2048,
@@ -332,7 +326,6 @@ fn test_instances_without_reuse_are_not_leaked() {
 				parallel_compilation: true,
 			},
 		},
-		sp_io::SubstrateHostFunctions::host_functions(),
 	)
 	.unwrap();