Extract execution engines definitions into their own crates (#4489)

* Clean imports in wasmi_execution * Replace `interpret_runtime_api_result` with `pointer_and_len_from_u64`. * Extract sc-executor-common crate * Extract `sc-executor-wasmi` into its own crate * Extract `sc-executor-wasmtime` into its own crate. * Add missing headers. * Clean and docs * Docs for sc-executor-wasmi * Expand a comment about sandboxing * Fix assert_matches * Rename (un)pack_ptr_and_len and move them into util module * Remove wasmtime errors in sc-executor-common
2026-06-11 23:31:07 +00:00 · 2019-12-24 13:17:41 +01:00
parent b214b3f3e9
commit 1782fbbbba
25 changed files with 326 additions and 161 deletions
@@ -19,19 +19,13 @@ lazy_static = "1.4.0"
 sp-wasm-interface = { version = "2.0.0", path = "../../primitives/wasm-interface" }
 sp-runtime-interface = { version = "2.0.0", path = "../../primitives/runtime-interface" }
 sp-externalities = { version = "2.0.0", path = "../../primitives/externalities" }
+sc-executor-common = { version = "2.0.0", path = "common" }
+sc-executor-wasmi = { version = "2.0.0", path = "wasmi" }
+sc-executor-wasmtime = { version = "2.0.0", path = "wasmtime", optional = true }
 parking_lot = "0.9.0"
 log = "0.4.8"
 libsecp256k1 = "0.3.2"

-cranelift-codegen = { version = "0.50", optional = true }
-cranelift-entity = { version = "0.50", optional = true }
-cranelift-frontend = { version = "0.50", optional = true }
-cranelift-native = { version = "0.50", optional = true }
-cranelift-wasm = { version = "0.50", optional = true }
-wasmtime-environ = { version = "0.8", optional = true }
-wasmtime-jit = { version = "0.8", optional = true }
-wasmtime-runtime = { version = "0.8", optional = true }
-
 [dev-dependencies]
 assert_matches = "1.3.0"
 wabt = "0.9.2"
@@ -47,14 +41,7 @@ default = [ "std" ]
 std = []
 wasm-extern-trace = []
 wasmtime = [
-	"cranelift-codegen",
-	"cranelift-entity",
-	"cranelift-frontend",
-	"cranelift-native",
-	"cranelift-wasm",
-	"wasmtime-environ",
-	"wasmtime-jit",
-	"wasmtime-runtime",
+	"sc-executor-wasmtime",
 ]
 wasmi-errno = [
 	"wasmi/errno"
@@ -0,0 +1,18 @@
+[package]
+name = "sc-executor-common"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+derive_more = "0.99.2"
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+wasmi = "0.6.2"
+sp-core = { version = "2.0.0", path = "../../../primitives/core" }
+sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" }
+sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" }
+sp-serializer = { version = "2.0.0", path = "../../../primitives/serializer" }
+
+[features]
+default = []
@@ -69,6 +69,9 @@ const MIN_POSSIBLE_ALLOCATION: u32 = 8;
 // to which it belongs.
 const PREFIX_SIZE: u32 = 8;

+/// An implementation of freeing bump allocator.
+///
+/// Refer to the module-level documentation for further details.
 pub struct FreeingBumpHeapAllocator {
 	bumper: u32,
 	heads: [u32; N],
@@ -18,8 +18,6 @@

 use sp_serializer;
 use wasmi;
-#[cfg(feature = "wasmtime")]
-use wasmtime_jit::{ActionError, SetupError};

 /// Result type alias.
 pub type Result<T> = std::result::Result<T, Error>;
@@ -33,9 +31,6 @@ pub enum Error {
 	Trap(wasmi::Trap),
 	/// Wasmi loading/instantiating error
 	Wasmi(wasmi::Error),
-	/// Wasmtime action error
-	#[cfg(feature = "wasmtime")]
-	Wasmtime(ActionError),
 	/// Error in the API. Parameter is an error message.
 	#[from(ignore)]
 	ApiError(String),
@@ -135,10 +130,6 @@ pub enum WasmError {
 	InvalidHeapPages,
 	/// Instantiation error.
 	Instantiation(String),
-	/// The compiler does not support the host machine as a target.
-	#[cfg(feature = "wasmtime")]
-	MissingCompilerSupport(&'static str),
-	/// Wasmtime setup error.
-	#[cfg(feature = "wasmtime")]
-	WasmtimeSetup(SetupError),
+	/// Other error happenend.
+	Other(String),
 }
@@ -0,0 +1,24 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A set of common definitions that are needed for defining execution engines.
+
+#![warn(missing_docs)]
+
+pub mod sandbox;
+pub mod allocator;
+pub mod error;
+pub mod wasm_runtime;
@@ -14,9 +14,10 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-#![warn(missing_docs)]
-
 //! This module implements sandboxing support in the runtime.
+//!
+//! Sandboxing is baked by wasmi at the moment. In future, however, we would like to add/switch to
+//! a compiled execution engine.

 use crate::error::{Result, Error};
 use std::{collections::HashMap, rc::Rc};
@@ -0,0 +1,39 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Definitions for a wasm runtime.
+
+use crate::error::Error;
+use sp_core::traits::Externalities;
+use sp_wasm_interface::Function;
+
+/// A trait that defines an abstract wasm runtime.
+///
+/// This can be implemented by an execution engine.
+pub trait WasmRuntime {
+	/// Attempt to update the number of heap pages available during execution.
+	///
+	/// Returns false if the update cannot be applied. The function is guaranteed to return true if
+	/// the heap pages would not change from its current value.
+	fn update_heap_pages(&mut self, heap_pages: u64) -> bool;
+
+	/// Return the host functions that are registered for this Wasm runtime.
+	fn host_functions(&self) -> &[&'static dyn Function];
+
+	/// Call a method in the Substrate runtime by name. Returns the encoded result on success.
+	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8])
+		-> Result<Vec<u8>, Error>;
+}
@@ -31,19 +31,13 @@

 #[macro_use]
 mod wasm_utils;
-mod wasmi_execution;
 #[macro_use]
 mod native_executor;
-mod sandbox;
-mod allocator;
 pub mod deprecated_host_interface;
 mod wasm_runtime;
-#[cfg(feature = "wasmtime")]
-mod wasmtime;
 #[cfg(test)]
 mod integration_tests;

-pub mod error;
 pub use wasmi;
 pub use native_executor::{with_native_environment, NativeExecutor, NativeExecutionDispatch};
 pub use sp_version::{RuntimeVersion, NativeVersion};
@@ -54,6 +48,8 @@ pub use sp_core::traits::Externalities;
 pub use sp_wasm_interface;
 pub use wasm_runtime::WasmExecutionMethod;

+pub use sc_executor_common::{error, allocator, sandbox};
+
 /// Call the given `function` in the given wasm `code`.
 ///
 /// The signature of `function` needs to follow the default Substrate function signature.
@@ -16,20 +16,15 @@

 use crate::{
 	RuntimeInfo, error::{Error, Result},
-	wasm_runtime::{RuntimesCache, WasmExecutionMethod, WasmRuntime},
+	wasm_runtime::{RuntimesCache, WasmExecutionMethod},
 };
-
 use sp_version::{NativeVersion, RuntimeVersion};
-
 use codec::{Decode, Encode};
-
 use sp_core::{NativeOrEncoded, traits::{CodeExecutor, Externalities}};
-
 use log::trace;
-
 use std::{result, cell::RefCell, panic::{UnwindSafe, AssertUnwindSafe}};
-
 use sp_wasm_interface::{HostFunctions, Function};
+use sc_executor_common::wasm_runtime::WasmRuntime;

 thread_local! {
 	static RUNTIMES_CACHE: RefCell<RuntimesCache> = RefCell::new(RuntimesCache::new());
@@ -19,36 +19,16 @@
 //! The primary means of accessing the runtimes is through a cache which saves the reusable
 //! components of the runtime that are expensive to initialize.

-use crate::{wasmi_execution, error::{Error, WasmError}};
-#[cfg(feature = "wasmtime")]
-use crate::wasmtime;
+use crate::error::{Error, WasmError};
 use log::{trace, warn};
-
 use codec::Decode;
-
 use sp_core::{storage::well_known_keys, traits::Externalities};
-
 use sp_version::RuntimeVersion;
 use std::{collections::hash_map::{Entry, HashMap}, panic::AssertUnwindSafe};
+use sc_executor_common::wasm_runtime::WasmRuntime;

 use sp_wasm_interface::Function;

-/// The Substrate Wasm runtime.
-pub trait WasmRuntime {
-	/// Attempt to update the number of heap pages available during execution.
-	///
-	/// Returns false if the update cannot be applied. The function is guaranteed to return true if
-	/// the heap pages would not change from its current value.
-	fn update_heap_pages(&mut self, heap_pages: u64) -> bool;
-
-	/// Return the host functions that are registered for this Wasm runtime.
-	fn host_functions(&self) -> &[&'static dyn Function];
-
-	/// Call a method in the Substrate runtime by name. Returns the encoded result on success.
-	fn call(&mut self, ext: &mut dyn Externalities, method: &str, data: &[u8])
-		-> Result<Vec<u8>, Error>;
-}
-
 /// Specification of different methods of executing the runtime Wasm code.
 #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)]
 pub enum WasmExecutionMethod {
@@ -214,11 +194,11 @@ pub fn create_wasm_runtime_with_code(
 ) -> Result<Box<dyn WasmRuntime>, WasmError> {
 	match wasm_method {
 		WasmExecutionMethod::Interpreted =>
-			wasmi_execution::create_instance(code, heap_pages, host_functions)
+			sc_executor_wasmi::create_instance(code, heap_pages, host_functions)
 				.map(|runtime| -> Box<dyn WasmRuntime> { Box::new(runtime) }),
 		#[cfg(feature = "wasmtime")]
 		WasmExecutionMethod::Compiled =>
-			wasmtime::create_instance(code, heap_pages, host_functions)
+			sc_executor_wasmtime::create_instance(code, heap_pages, host_functions)
 				.map(|runtime| -> Box<dyn WasmRuntime> { Box::new(runtime) }),
 	}
 }
@@ -16,8 +16,6 @@

 //! Utilities for defining the wasm host environment.

-use sp_wasm_interface::{Pointer, WordSize};
-
 /// Converts arguments into respective WASM types.
 #[macro_export]
 macro_rules! convert_args {
@@ -173,14 +171,3 @@ macro_rules! impl_wasm_host_interface {
 		}
 	);
 }
-
-/// Runtime API functions return an i64 which encodes a pointer in the least-significant 32 bits
-/// and a length in the most-significant 32 bits. This interprets the returned value as a pointer,
-/// length tuple.
-pub fn interpret_runtime_api_result(retval: i64) -> (Pointer<u8>, WordSize) {
-	let ptr = <Pointer<u8>>::new(retval as u32);
-	// The first cast to u64 is necessary so that the right shift does not sign-extend.
-	let len = ((retval as u64) >> 32) as WordSize;
-	(ptr, len)
-}
-
@@ -0,0 +1,16 @@
+[package]
+name = "sc-executor-wasmi"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+wasmi = "0.6.2"
+parity-wasm = "0.41.0"
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+sc-executor-common = { version = "2.0.0", path = "../common" }
+sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" }
+sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" }
+sp-core = { version = "2.0.0", path = "../../../primitives/core" }
+sp-externalities = { version = "2.0.0", path = "../../../primitives/externalities" }
@@ -1,4 +1,4 @@
-// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// Copyright 2019 Parity Technologies (UK) Ltd.
 // This file is part of Substrate.

 // Substrate is free software: you can redistribute it and/or modify
@@ -14,25 +14,27 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-//! Implementation of a Wasm runtime using the Wasmi interpreter.
+//! This crate provides an implementation of `WasmRuntime` that is baked by wasmi.

+use sc_executor_common::{
+	error::{Error, WasmError},
+	sandbox,
+	allocator,
+};
 use std::{str, mem};
 use wasmi::{
 	Module, ModuleInstance, MemoryInstance, MemoryRef, TableRef, ImportsBuilder, ModuleRef,
 	memory_units::Pages, RuntimeValue::{I32, I64, self},
 };
-use crate::error::{Error, WasmError};
 use codec::{Encode, Decode};
 use sp_core::{sandbox as sandbox_primitives, traits::Externalities};
-use crate::sandbox;
-use crate::allocator;
-use crate::wasm_utils::interpret_runtime_api_result;
-use crate::wasm_runtime::WasmRuntime;
 use log::{error, trace};
 use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule};
 use sp_wasm_interface::{
 	FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function,
 };
+use sp_runtime_interface::unpack_ptr_and_len;
+use sc_executor_common::wasm_runtime::WasmRuntime;

 struct FunctionExecutor<'a> {
 	sandbox_store: sandbox::Store<wasmi::FuncRef>,
@@ -375,7 +377,7 @@ fn call_in_wasm_module(

 	match result {
 		Ok(Some(I64(r))) => {
-			let (ptr, length) = interpret_runtime_api_result(r);
+			let (ptr, length) = unpack_ptr_and_len(r as u64);
 			memory.get(ptr.into(), length as usize).map_err(|_| Error::Runtime)
 		},
 		Err(e) => {
@@ -0,0 +1,28 @@
+[package]
+name = "sc-executor-wasmtime"
+version = "2.0.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+wasmi = "0.6.2"
+parity-wasm = "0.41.0"
+codec = { package = "parity-scale-codec", version = "1.0.0" }
+sc-executor-common = { version = "2.0.0", path = "../common" }
+sp-wasm-interface = { version = "2.0.0", path = "../../../primitives/wasm-interface" }
+sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime-interface" }
+sp-core = { version = "2.0.0", path = "../../../primitives/core" }
+sp-externalities = { version = "2.0.0", path = "../../../primitives/externalities" }
+
+cranelift-codegen = "0.50"
+cranelift-entity = "0.50"
+cranelift-frontend = "0.50"
+cranelift-native = "0.50"
+cranelift-wasm = "0.50"
+wasmtime-environ = "0.8"
+wasmtime-jit = "0.8"
+wasmtime-runtime = "0.8"
+
+[dev-dependencies]
+assert_matches = "1.3.0"
@@ -14,10 +14,10 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-use crate::allocator::FreeingBumpHeapAllocator;
-use crate::error::{Error, Result};
-use crate::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex};
-use crate::wasmtime::util::{
+use sc_executor_common::allocator::FreeingBumpHeapAllocator;
+use sc_executor_common::error::{Error, Result};
+use sc_executor_common::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex};
+use crate::util::{
 	checked_range, cranelift_ir_signature, read_memory_into, write_memory_from,
 };

@@ -173,7 +173,7 @@ impl<'a> SandboxCapabilities for FunctionExecutor<'a> {
 		let exec_code_buf = self.compiler
 			.get_published_trampoline(func_ptr, &signature, value_size)
 			.map_err(ActionError::Setup)
-			.map_err(Error::Wasmtime)?;
+			.map_err(|e| Error::Other(e.to_string()))?;

 		// Call the trampoline.
 		if let Err(message) = unsafe {
@@ -22,3 +22,4 @@ mod trampoline;
 mod util;

 pub use runtime::create_instance;
+
@@ -16,28 +16,31 @@

 //! Defines the compiled Wasm runtime that uses Wasmtime internally.

-use crate::error::{Error, Result, WasmError};
-use crate::wasm_runtime::WasmRuntime;
-use crate::wasm_utils::interpret_runtime_api_result;
-use crate::wasmtime::function_executor::FunctionExecutorState;
-use crate::wasmtime::trampoline::{EnvState, make_trampoline};
-use crate::wasmtime::util::{cranelift_ir_signature, read_memory_into, write_memory_from};
-use crate::Externalities;
+use crate::function_executor::FunctionExecutorState;
+use crate::trampoline::{EnvState, make_trampoline};
+use crate::util::{cranelift_ir_signature, read_memory_into, write_memory_from};
+
+use sc_executor_common::{
+	error::{Error, Result, WasmError},
+	wasm_runtime::WasmRuntime,
+};
+use sp_core::traits::Externalities;
+use sp_wasm_interface::{Pointer, WordSize, Function};
+use sp_runtime_interface::unpack_ptr_and_len;
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::rc::Rc;

 use cranelift_codegen::ir;
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_entity::{EntityRef, PrimaryMap};
 use cranelift_frontend::FunctionBuilderContext;
 use cranelift_wasm::DefinedFuncIndex;
-use std::cell::RefCell;
-use std::collections::HashMap;
-use std::convert::TryFrom;
-use std::rc::Rc;
-use sp_wasm_interface::{Pointer, WordSize, Function};
 use wasmtime_environ::{Module, translate_signature};
 use wasmtime_jit::{
-	ActionOutcome, ActionError, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context,
-	SetupError, RuntimeValue,
+	ActionOutcome, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, RuntimeValue,
 };
 use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody};

@@ -134,7 +137,7 @@ fn create_compiled_unit(

 	// Compile the wasm module.
 	let module = context.compile_module(&code)
-		.map_err(WasmError::WasmtimeSetup)?;
+		.map_err(|e| WasmError::Other(format!("module compile error: {}", e)))?;

 	Ok((module, context))
 }
@@ -155,9 +158,7 @@ fn call_method(
 	clear_globals(&mut *context.get_global_exports().borrow_mut());

 	let mut instance = module.instantiate()
-		.map_err(SetupError::Instantiate)
-		.map_err(ActionError::Setup)
-		.map_err(Error::Wasmtime)?;
+		.map_err(|e| Error::Other(e.to_string()))?;

 	// Ideally there would be a way to set the heap pages during instantiation rather than
 	// growing the memory after the fact. Currently this may require an additional mmap and copy.
@@ -178,12 +179,12 @@ fn call_method(
 	let outcome = sp_externalities::set_and_run_with_externalities(ext, || {
 		context
 			.invoke(&mut instance, method, &args[..])
-			.map_err(Error::Wasmtime)
+			.map_err(|e| Error::Other(format!("error calling runtime: {}", e)))
 	})?;
 	let trap_error = reset_env_state_and_take_trap(context, None)?;
 	let (output_ptr, output_len) = match outcome {
 		ActionOutcome::Returned { values } => match values.as_slice() {
-			[RuntimeValue::I64(retval)] => interpret_runtime_api_result(*retval),
+			[RuntimeValue::I64(retval)] => unpack_ptr_and_len(*retval as u64),
 			_ => return Err(Error::InvalidReturn),
 		}
 		ActionOutcome::Trapped { message } => return Err(trap_error.unwrap_or_else(
@@ -194,7 +195,7 @@ fn call_method(
 	// Read the output data from guest memory.
 	let mut output = vec![0; output_len as usize];
 	let memory = get_memory_mut(&mut instance)?;
-	read_memory_into(memory, output_ptr, &mut output)?;
+	read_memory_into(memory, Pointer::new(output_ptr), &mut output)?;
 	Ok(output)
 }

@@ -252,13 +253,13 @@ fn instantiate_env_module(
 		None,
 		Box::new(env_state),
 	);
-	result.map_err(|e| WasmError::WasmtimeSetup(SetupError::Instantiate(e)))
+	result.map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e)))
 }

 /// Build a new TargetIsa for the host machine.
 fn target_isa() -> std::result::Result<Box<dyn TargetIsa>, WasmError> {
 	let isa_builder = cranelift_native::builder()
-		.map_err(WasmError::MissingCompilerSupport)?;
+		.map_err(|e| WasmError::Other(format!("missing compiler support: {}", e)))?;
 	let flag_builder = cranelift_codegen::settings::builder();
 	Ok(isa_builder.finish(cranelift_codegen::settings::Flags::new(flag_builder)))
 }
@@ -19,6 +19,9 @@
 //! This code is based on and large parts are copied from wasmtime's
 //! wasmtime-api/src/trampoline/func.rs.

+use crate::function_executor::{FunctionExecutorState, FunctionExecutor};
+use sc_executor_common::error::{Error, WasmError};
+
 use cranelift_codegen::{Context, binemit, ir, isa};
 use cranelift_codegen::ir::{InstBuilder, StackSlotData, StackSlotKind, TrapCode};
 use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
@@ -29,9 +32,6 @@ use wasmtime_runtime::{VMContext, VMFunctionBody};
 use sp_wasm_interface::{Function, Value, ValueType};
 use std::{cmp, panic::{self, AssertUnwindSafe}, ptr};

-use crate::error::{Error, WasmError};
-use crate::wasmtime::function_executor::{FunctionExecutorState, FunctionExecutor};
-
 const CALL_SUCCESS: u32 = 0;
 const CALL_FAILED_WITH_ERROR: u32 = 1;
 const CALL_WITH_BAD_HOST_STATE: u32 = 2;
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-use crate::error::{Error, Result};
+use sc_executor_common::error::{Error, Result};

 use cranelift_codegen::{ir, isa};
 use std::ops::Range;