WASM runtime switch to import memory (#4737)

* WASM runtime switch to import memory

Up to now runtimes have exported their memory. To unify it with
sandboxing, this pr switches runtimes to import memory as well.

From a functional perspective, exporting/importing memory makes no
difference to the runtime.

To provide backwards compatibility, WASM exported memory is still supported.

* Revert debug stuff

* Revert some stuff
This commit is contained in:
Bastian Köcher
2020-01-28 09:36:57 +01:00
committed by GitHub
parent 5c8743510e
commit 793a1eb053
15 changed files with 603 additions and 428 deletions
+6 -6
View File
@@ -3234,7 +3234,7 @@ dependencies = [
"sp-std 2.0.0",
"sp-transaction-pool 2.0.0",
"sp-version 2.0.0",
"substrate-wasm-builder-runner 1.0.4",
"substrate-wasm-builder-runner 1.0.5",
]
[[package]]
@@ -3295,7 +3295,7 @@ dependencies = [
"sp-std 2.0.0",
"sp-transaction-pool 2.0.0",
"sp-version 2.0.0",
"substrate-wasm-builder-runner 1.0.4",
"substrate-wasm-builder-runner 1.0.5",
]
[[package]]
@@ -5823,7 +5823,7 @@ dependencies = [
"sp-runtime 2.0.0",
"sp-sandbox 0.8.0",
"sp-std 2.0.0",
"substrate-wasm-builder-runner 1.0.4",
"substrate-wasm-builder-runner 1.0.5",
]
[[package]]
@@ -6693,7 +6693,7 @@ dependencies = [
"sp-io 2.0.0",
"sp-runtime-interface 2.0.0",
"sp-std 2.0.0",
"substrate-wasm-builder-runner 1.0.4",
"substrate-wasm-builder-runner 1.0.5",
]
[[package]]
@@ -7063,7 +7063,7 @@ dependencies = [
"sp-trie 2.0.0",
"sp-version 2.0.0",
"substrate-test-runtime-client 2.0.0",
"substrate-wasm-builder-runner 1.0.4",
"substrate-wasm-builder-runner 1.0.5",
"trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -7104,7 +7104,7 @@ dependencies = [
[[package]]
name = "substrate-wasm-builder-runner"
version = "1.0.4"
version = "1.0.5"
[[package]]
name = "subtle"
+7 -8
View File
@@ -1,11 +1,10 @@
use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
use wasm_builder_runner::WasmBuilder;
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::Crates("1.0.9"),
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
"-Clink-arg=--export=__heap_base",
);
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_from_crates("1.0.9")
.export_heap_base()
.import_memory()
.build()
}
+7 -11
View File
@@ -14,17 +14,13 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
use wasm_builder_runner::WasmBuilder;
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../../utils/wasm-builder",
version: "1.0.9",
},
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
"-Clink-arg=--export=__heap_base",
);
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder")
.export_heap_base()
.import_memory()
.build()
}
@@ -14,17 +14,13 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
use wasm_builder_runner::WasmBuilder;
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../../utils/wasm-builder",
version: "1.0.9",
},
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
"-Clink-arg=--export=__heap_base",
);
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder")
.export_heap_base()
.import_memory()
.build()
}
@@ -26,6 +26,11 @@ extern "C" {
fn yet_another_missing_external();
}
#[cfg(not(feature = "std"))]
/// Mutable static variables should be always observed to have
/// the initialized value at the start of a runtime call.
static mut MUTABLE_STATIC: u64 = 32;
sp_core::wasm_export_functions! {
fn test_calling_missing_external() {
unsafe { missing_external() }
@@ -217,6 +222,41 @@ sp_core::wasm_export_functions! {
fn test_sp_allocator_compiles() {
sp_allocator::FreeingBumpHeapAllocator::new(0);
}
fn returns_mutable_static() -> u64 {
unsafe {
MUTABLE_STATIC += 1;
MUTABLE_STATIC
}
}
fn allocates_huge_stack_array(trap: bool) -> Vec<u8> {
// Allocate a stack frame that is approx. 75% of the stack (assuming it is 1MB).
// This will just decrease (stacks in wasm32-u-u grow downwards) the stack
// pointer. This won't trap on the current compilers.
let mut data = [0u8; 1024 * 768];
// Then make sure we actually write something to it.
//
// If:
// 1. the stack area is placed at the beginning of the linear memory space, and
// 2. the stack pointer points to out-of-bounds area, and
// 3. a write is performed around the current stack pointer.
//
// then a trap should happen.
//
for (i, v) in data.iter_mut().enumerate() {
*v = i as u8; // deliberate truncation
}
if trap {
// There is a small chance of this to be pulled up in theory. In practice
// the probability of that is rather low.
panic!()
}
data.to_vec()
}
}
#[cfg(not(feature = "std"))]
@@ -27,6 +27,7 @@ use sc_runtime_test::WASM_BINARY;
use sp_state_machine::TestExternalities as CoreTestExternalities;
use test_case::test_case;
use sp_trie::{TrieConfiguration, trie_types::Layout};
use sp_wasm_interface::HostFunctions as _;
use crate::WasmExecutionMethod;
@@ -38,16 +39,14 @@ fn call_in_wasm<E: Externalities>(
call_data: &[u8],
execution_method: WasmExecutionMethod,
ext: &mut E,
code: &[u8],
heap_pages: u64,
) -> crate::error::Result<Vec<u8>> {
crate::call_in_wasm::<HostFunctions>(
function,
call_data,
execution_method,
ext,
code,
heap_pages,
&WASM_BINARY[..],
1024,
true,
)
}
@@ -57,15 +56,12 @@ fn call_in_wasm<E: Externalities>(
fn returning_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let output = call_in_wasm(
"test_empty_return",
&[],
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap();
assert_eq!(output, vec![0u8; 0]);
}
@@ -73,17 +69,14 @@ fn returning_should_work(wasm_method: WasmExecutionMethod) {
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
fn call_not_existing_function(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
match call_in_wasm(
"test_calling_missing_external",
&[],
wasm_method,
&mut ext,
&test_code[..],
8,
) {
Ok(_) => panic!("was expected an `Err`"),
Err(e) => {
@@ -107,15 +100,12 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) {
fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
match call_in_wasm(
"test_calling_yet_another_missing_external",
&[],
wasm_method,
&mut ext,
&test_code[..],
8,
) {
Ok(_) => panic!("was expected an `Err`"),
Err(e) => {
@@ -139,15 +129,12 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) {
fn panicking_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let output = call_in_wasm(
"test_panic",
&[],
wasm_method,
&mut ext,
&test_code[..],
8,
);
assert!(output.is_err());
@@ -156,8 +143,6 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) {
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
);
assert_eq!(Decode::decode(&mut &output.unwrap()[..]), Ok(Vec::<u8>::new()));
@@ -166,8 +151,6 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) {
&vec![2].encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
);
assert!(output.is_err());
}
@@ -180,15 +163,12 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) {
{
let mut ext = ext.ext();
ext.set_storage(b"foo".to_vec(), b"bar".to_vec());
let test_code = WASM_BINARY;
let output = call_in_wasm(
"test_data_in",
&b"Hello world".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap();
assert_eq!(output, b"all ok!".to_vec().encode());
@@ -216,7 +196,6 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) {
ext.set_storage(b"aba".to_vec(), b"3".to_vec());
ext.set_storage(b"abb".to_vec(), b"4".to_vec());
ext.set_storage(b"bbb".to_vec(), b"5".to_vec());
let test_code = WASM_BINARY;
// This will clear all entries which prefix is "ab".
let output = call_in_wasm(
@@ -224,8 +203,6 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) {
&b"ab".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap();
assert_eq!(output, b"all ok!".to_vec().encode());
@@ -247,15 +224,12 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) {
fn blake2_256_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_blake2_256",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
blake2_256(&b""[..]).to_vec().encode(),
);
@@ -265,8 +239,6 @@ fn blake2_256_should_work(wasm_method: WasmExecutionMethod) {
&b"Hello world!".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
blake2_256(&b"Hello world!"[..]).to_vec().encode(),
);
@@ -277,15 +249,12 @@ fn blake2_256_should_work(wasm_method: WasmExecutionMethod) {
fn blake2_128_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_blake2_128",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
blake2_128(&b""[..]).to_vec().encode(),
);
@@ -295,8 +264,6 @@ fn blake2_128_should_work(wasm_method: WasmExecutionMethod) {
&b"Hello world!".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
blake2_128(&b"Hello world!"[..]).to_vec().encode(),
);
@@ -307,15 +274,12 @@ fn blake2_128_should_work(wasm_method: WasmExecutionMethod) {
fn sha2_256_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_sha2_256",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
)
.unwrap(),
hex!("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
@@ -328,8 +292,6 @@ fn sha2_256_should_work(wasm_method: WasmExecutionMethod) {
&b"Hello world!".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
)
.unwrap(),
hex!("c0535e4be2b79ffd93291305436bf889314e4a3faec05ecffcbb7df31ad9e51a")
@@ -343,15 +305,12 @@ fn sha2_256_should_work(wasm_method: WasmExecutionMethod) {
fn twox_256_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_twox_256",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
hex!(
"99e9d85137db46ef4bbea33613baafd56f963c64b1f3685a4eb4abd67ff6203a"
@@ -363,8 +322,6 @@ fn twox_256_should_work(wasm_method: WasmExecutionMethod) {
&b"Hello world!".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
hex!(
"b27dfd7f223f177f2a13647b533599af0c07f68bda23d96d059da2b451a35a74"
@@ -377,15 +334,12 @@ fn twox_256_should_work(wasm_method: WasmExecutionMethod) {
fn twox_128_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_twox_128",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(),
);
@@ -395,8 +349,6 @@ fn twox_128_should_work(wasm_method: WasmExecutionMethod) {
&b"Hello world!".to_vec().encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
hex!("b27dfd7f223f177f2a13647b533599af").to_vec().encode(),
);
@@ -407,7 +359,6 @@ fn twox_128_should_work(wasm_method: WasmExecutionMethod) {
fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let key = ed25519::Pair::from_seed(&blake2_256(b"test"));
let sig = key.sign(b"all ok!");
let mut calldata = vec![];
@@ -420,8 +371,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) {
&calldata.encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -437,8 +386,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) {
&calldata.encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
false.encode(),
);
@@ -449,7 +396,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) {
fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let key = sr25519::Pair::from_seed(&blake2_256(b"test"));
let sig = key.sign(b"all ok!");
let mut calldata = vec![];
@@ -462,8 +408,6 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) {
&calldata.encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -479,8 +423,6 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) {
&calldata.encode(),
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
false.encode(),
);
@@ -490,17 +432,13 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) {
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()];
let test_code = WASM_BINARY;
assert_eq!(
call_in_wasm(
"test_ordered_trie_root",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
&mut ext.ext(),
).unwrap(),
Layout::<Blake2Hasher>::ordered_trie_root(trie_input.iter()).as_bytes().encode(),
);
@@ -514,16 +452,12 @@ fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let (offchain, state) = testing::TestOffchainExt::new();
ext.register_extension(OffchainExt::new(offchain));
let test_code = WASM_BINARY;
let mut ext = ext.ext();
assert_eq!(
call_in_wasm(
"test_offchain_local_storage",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
&mut ext.ext(),
).unwrap(),
true.encode(),
);
@@ -550,17 +484,84 @@ fn offchain_http_should_work(wasm_method: WasmExecutionMethod) {
},
);
let test_code = WASM_BINARY;
let mut ext = ext.ext();
assert_eq!(
call_in_wasm(
"test_offchain_http",
&[0],
wasm_method,
&mut ext,
&test_code[..],
8,
&mut ext.ext(),
).unwrap(),
true.encode(),
);
}
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
#[should_panic(expected = "Allocator ran out of space")]
fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
crate::call_in_wasm::<HostFunctions>(
"test_exhaust_heap",
&[0],
wasm_method,
&mut ext.ext(),
&WASM_BINARY[..],
// `17` is the initial number of pages compiled into the binary.
17,
true,
).unwrap();
}
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code(
wasm_method,
1024,
&WASM_BINARY[..],
HostFunctions::host_functions(),
true,
).expect("Creates instance");
let res = instance.call("returns_mutable_static", &[0]).unwrap();
assert_eq!(33, u64::decode(&mut &res[..]).unwrap());
// We expect that every invocation will need to return the initial
// value plus one. If the value increases more than that then it is
// a sign that the wasm runtime preserves the memory content.
let res = instance.call("returns_mutable_static", &[0]).unwrap();
assert_eq!(33, u64::decode(&mut &res[..]).unwrap());
}
// If we didn't restore the wasm instance properly, on a trap the stack pointer would not be
// returned to its initial value and thus the stack space is going to be leaked.
//
// See https://github.com/paritytech/substrate/issues/2967 for details
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
// Allocate 32 pages (of 65536 bytes) which gives the runtime 2048KB of heap to operate on
// (plus some additional space unused from the initial pages requested by the wasm runtime
// module).
//
// The fixture performs 2 allocations of 768KB and this theoretically gives 1536KB, however, due
// to our allocator algorithm there are inefficiencies.
const REQUIRED_MEMORY_PAGES: u64 = 32;
let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code(
wasm_method,
REQUIRED_MEMORY_PAGES,
&WASM_BINARY[..],
HostFunctions::host_functions(),
true,
).expect("Creates instance");
// On the first invocation we allocate approx. 768KB (75%) of stack and then trap.
let res = instance.call("allocates_huge_stack_array", &true.encode());
assert!(res.is_err());
// On the second invocation we allocate yet another 768KB (75%) of stack
let res = instance.call("allocates_huge_stack_array", &false.encode());
assert!(res.is_ok());
}
@@ -18,7 +18,6 @@ use super::{TestExternalities, call_in_wasm};
use crate::WasmExecutionMethod;
use codec::Encode;
use sc_runtime_test::WASM_BINARY;
use test_case::test_case;
use wabt;
@@ -27,7 +26,6 @@ use wabt;
fn sandbox_should_work(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -56,8 +54,6 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -68,7 +64,6 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) {
fn sandbox_trap(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -86,47 +81,16 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
vec![0],
);
}
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
#[should_panic(expected = "Allocator ran out of space")]
fn sandbox_should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
(import "env" "assert" (func $assert (param i32)))
(func (export "call")
i32.const 0
call $assert
)
)
"#).unwrap().encode();
call_in_wasm(
"test_exhaust_heap",
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap();
}
#[test_case(WasmExecutionMethod::Interpreted)]
#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))]
fn start_called(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -161,8 +125,6 @@ fn start_called(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -173,7 +135,6 @@ fn start_called(wasm_method: WasmExecutionMethod) {
fn invoke_args(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -204,8 +165,6 @@ fn invoke_args(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -216,7 +175,6 @@ fn invoke_args(wasm_method: WasmExecutionMethod) {
fn return_val(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -235,8 +193,6 @@ fn return_val(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
true.encode(),
);
@@ -247,7 +203,6 @@ fn return_val(wasm_method: WasmExecutionMethod) {
fn unlinkable_module(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -264,8 +219,6 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
1u8.encode(),
);
@@ -276,7 +229,6 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) {
fn corrupted_module(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
// Corrupted wasm file
let code = vec![0u8, 0, 0, 0, 1, 0, 0, 0].encode();
@@ -287,8 +239,6 @@ fn corrupted_module(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
1u8.encode(),
);
@@ -299,7 +249,6 @@ fn corrupted_module(wasm_method: WasmExecutionMethod) {
fn start_fn_ok(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -319,8 +268,6 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
0u8.encode(),
);
@@ -331,7 +278,6 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) {
fn start_fn_traps(wasm_method: WasmExecutionMethod) {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
let test_code = WASM_BINARY;
let code = wabt::wat2wasm(r#"
(module
@@ -352,8 +298,6 @@ fn start_fn_traps(wasm_method: WasmExecutionMethod) {
&code,
wasm_method,
&mut ext,
&test_code[..],
8,
).unwrap(),
2u8.encode(),
);
+96 -22
View File
@@ -24,7 +24,7 @@ use wasmi::{
};
use codec::{Encode, Decode};
use sp_core::sandbox as sandbox_primitives;
use log::{error, trace};
use log::{error, trace, debug};
use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule};
use sp_wasm_interface::{
FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function,
@@ -271,18 +271,36 @@ impl<'a> Sandbox for FunctionExecutor<'a> {
}
}
/// Will be used on initialization of a module to resolve function and memory imports.
struct Resolver<'a> {
host_functions: &'a[&'static dyn Function],
/// All the hot functions that we export for the WASM blob.
host_functions: &'a [&'static dyn Function],
/// Should we allow missing function imports?
///
/// If `true`, we return a stub that will return an error when being called.
allow_missing_func_imports: bool,
/// All the names of functions for that we did not provide a host function.
missing_functions: RefCell<Vec<String>>,
/// Will be used as initial and maximum size of the imported memory.
heap_pages: usize,
/// By default, runtimes should import memory and this is `Some(_)` after
/// reolving. However, to be backwards compatible, we also support memory
/// exported by the WASM blob (this will be `None` after resolving).
import_memory: RefCell<Option<MemoryRef>>,
}
impl<'a> Resolver<'a> {
fn new(host_functions: &'a[&'static dyn Function], allow_missing_func_imports: bool) -> Resolver<'a> {
fn new(
host_functions: &'a[&'static dyn Function],
allow_missing_func_imports: bool,
heap_pages: usize,
) -> Resolver<'a> {
Resolver {
host_functions,
allow_missing_func_imports,
missing_functions: RefCell::new(Vec::new()),
heap_pages,
import_memory: Default::default(),
}
}
}
@@ -323,6 +341,48 @@ impl<'a> wasmi::ModuleImportResolver for Resolver<'a> {
))
}
}
fn resolve_memory(
&self,
field_name: &str,
memory_type: &wasmi::MemoryDescriptor,
) -> Result<MemoryRef, wasmi::Error> {
if field_name == "memory" {
match &mut *self.import_memory.borrow_mut() {
Some(_) => Err(wasmi::Error::Instantiation(
"Memory can not be imported twice!".into(),
)),
memory_ref @ None => {
if memory_type
.maximum()
.map(|m| m.saturating_sub(memory_type.initial()))
.map(|m| self.heap_pages > m as usize)
.unwrap_or(false)
{
Err(wasmi::Error::Instantiation(format!(
"Heap pages ({}) is greater than imported memory maximum ({}).",
self.heap_pages,
memory_type
.maximum()
.map(|m| m.saturating_sub(memory_type.initial()))
.expect("Maximum is set, checked above; qed"),
)))
} else {
let memory = MemoryInstance::alloc(
Pages(memory_type.initial() as usize + self.heap_pages),
Some(Pages(memory_type.initial() as usize + self.heap_pages)),
)?;
*memory_ref = Some(memory.clone());
Ok(memory)
}
}
}
} else {
Err(wasmi::Error::Instantiation(
format!("Unknown memory reference with name: {}", field_name),
))
}
}
}
impl<'a> wasmi::Externals for FunctionExecutor<'a> {
@@ -378,15 +438,14 @@ fn get_heap_base(module: &ModuleRef) -> Result<u32, Error> {
/// Call a given method in the given wasm-module runtime.
fn call_in_wasm_module(
module_instance: &ModuleRef,
memory: &MemoryRef,
method: &str,
data: &[u8],
host_functions: &[&'static dyn Function],
allow_missing_func_imports: bool,
missing_functions: &Vec<String>,
) -> Result<Vec<u8>, Error> {
// extract a reference to a linear memory, optional reference to a table
// and then initialize FunctionExecutor.
let memory = get_mem_instance(module_instance)?;
// Initialize FunctionExecutor.
let table: Option<TableRef> = module_instance
.export_by_name("__indirect_function_table")
.and_then(|e| e.as_table().cloned());
@@ -434,8 +493,8 @@ fn instantiate_module(
module: &Module,
host_functions: &[&'static dyn Function],
allow_missing_func_imports: bool,
) -> Result<(ModuleRef, Vec<String>), Error> {
let resolver = Resolver::new(host_functions, allow_missing_func_imports);
) -> Result<(ModuleRef, Vec<String>, MemoryRef), Error> {
let resolver = Resolver::new(host_functions, allow_missing_func_imports, heap_pages);
// start module instantiation. Don't run 'start' function yet.
let intermediate_instance = ModuleInstance::new(
module,
@@ -445,15 +504,33 @@ fn instantiate_module(
// Verify that the module has the heap base global variable.
let _ = get_heap_base(intermediate_instance.not_started_instance())?;
// Extract a reference to a linear memory.
let memory = get_mem_instance(intermediate_instance.not_started_instance())?;
memory.grow(Pages(heap_pages)).map_err(|_| Error::Runtime)?;
// Get the memory reference. Runtimes should import memory, but to be backwards
// compatible we also support exported memory.
let memory = match resolver.import_memory.into_inner() {
Some(memory) => memory,
None => {
debug!(
target: "wasm-executor",
"WASM blob does not imports memory, falling back to exported memory",
);
let memory = get_mem_instance(intermediate_instance.not_started_instance())?;
memory.grow(Pages(heap_pages)).map_err(|_| Error::Runtime)?;
memory
}
};
if intermediate_instance.has_start() {
// Runtime is not allowed to have the `start` function.
Err(Error::RuntimeHasStartFn)
} else {
Ok((intermediate_instance.assert_no_start(), resolver.missing_functions.into_inner()))
Ok((
intermediate_instance.assert_no_start(),
resolver.missing_functions.into_inner(),
memory,
))
}
}
@@ -528,14 +605,7 @@ impl StateSnapshot {
/// the preserved memory and globals.
///
/// Returns `Err` if applying the snapshot is failed.
fn apply(&self, instance: &ModuleRef) -> Result<(), WasmError> {
let memory = instance
.export_by_name("memory")
.ok_or(WasmError::ApplySnapshotFailed)?
.as_memory()
.cloned()
.ok_or(WasmError::ApplySnapshotFailed)?;
fn apply(&self, instance: &ModuleRef, memory: &MemoryRef) -> Result<(), WasmError> {
// First, erase the memory and copy the data segments into it.
memory
.erase()
@@ -569,6 +639,8 @@ impl StateSnapshot {
pub struct WasmiRuntime {
/// A wasm module instance.
instance: ModuleRef,
/// The memory instance of used by the wasm module.
memory: MemoryRef,
/// The snapshot of the instance's state taken just after the instantiation.
state_snapshot: StateSnapshot,
/// The host functions registered for this instance.
@@ -594,7 +666,7 @@ impl WasmRuntime for WasmiRuntime {
method: &str,
data: &[u8],
) -> Result<Vec<u8>, Error> {
self.state_snapshot.apply(&self.instance)
self.state_snapshot.apply(&self.instance, &self.memory)
.map_err(|e| {
// Snapshot restoration failed. This is pretty unexpected since this can happen
// if some invariant is broken or if the system is under extreme memory pressure
@@ -604,6 +676,7 @@ impl WasmRuntime for WasmiRuntime {
})?;
call_in_wasm_module(
&self.instance,
&self.memory,
method,
data,
&self.host_functions,
@@ -628,7 +701,7 @@ pub fn create_instance(
let data_segments = extract_data_segments(&code)?;
// Instantiate this module.
let (instance, missing_functions) = instantiate_module(
let (instance, missing_functions, memory) = instantiate_module(
heap_pages as usize,
&module,
&host_functions,
@@ -646,6 +719,7 @@ pub fn create_instance(
Ok(WasmiRuntime {
instance,
memory,
state_snapshot,
host_functions,
allow_missing_func_imports,
+105 -71
View File
@@ -32,42 +32,38 @@ use sc_executor_common::{
use sp_wasm_interface::{Pointer, WordSize, Function};
use sp_runtime_interface::unpack_ptr_and_len;
use std::cell::RefCell;
use std::collections::HashMap;
use std::convert::TryFrom;
use std::rc::Rc;
use std::{cell::RefCell, collections::HashMap, convert::TryFrom, rc::Rc};
use cranelift_codegen::ir;
use cranelift_codegen::isa::TargetIsa;
use cranelift_entity::{EntityRef, PrimaryMap};
use cranelift_frontend::FunctionBuilderContext;
use cranelift_wasm::DefinedFuncIndex;
use cranelift_wasm::{DefinedFuncIndex, MemoryIndex};
use wasmtime_environ::{Module, translate_signature};
use wasmtime_jit::{
ActionOutcome, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, RuntimeValue,
};
use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody};
/// TODO: We should remove this in https://github.com/paritytech/substrate/pull/4686
/// Currently there is no way to extract this with wasmtime.
const INITIAL_HEAP_PAGES: u32 = 17;
/// A `WasmRuntime` implementation using the Wasmtime JIT to compile the runtime module to native
/// and execute the compiled code.
pub struct WasmtimeRuntime {
module: CompiledModule,
context: Context,
max_heap_pages: Option<u32>,
heap_pages: u32,
/// The host functions registered for this instance.
host_functions: Vec<&'static dyn Function>,
/// The index of the memory in the module.
memory_index: MemoryIndex,
}
impl WasmRuntime for WasmtimeRuntime {
fn update_heap_pages(&mut self, heap_pages: u64) -> bool {
match heap_pages_valid(heap_pages, self.max_heap_pages) {
Some(heap_pages) => {
self.heap_pages = heap_pages;
true
}
None => false,
}
self.heap_pages as u64 == heap_pages
}
fn host_functions(&self) -> &[&'static dyn Function] {
@@ -80,6 +76,7 @@ impl WasmRuntime for WasmtimeRuntime {
&mut self.module,
method,
data,
self.memory_index,
self.heap_pages,
)
}
@@ -93,31 +90,42 @@ pub fn create_instance(
host_functions: Vec<&'static dyn Function>,
allow_missing_func_imports: bool,
) -> std::result::Result<WasmtimeRuntime, WasmError> {
let (compiled_module, context) = create_compiled_unit(code, &host_functions, allow_missing_func_imports)?;
let heap_pages = u32::try_from(heap_pages)
.map_err(|e|
WasmError::Other(format!("Heap pages can not be converted into `u32`: {:?}", e))
)?;
// Inspect the module for the min and max memory sizes.
let (min_memory_size, max_memory_size) = {
let module = compiled_module.module_ref();
let memory_index = match module.exports.get("memory") {
Some(wasmtime_environ::Export::Memory(memory_index)) => *memory_index,
_ => return Err(WasmError::InvalidMemory),
let (compiled_module, context, memory_index) = create_compiled_unit(
code,
&host_functions,
heap_pages,
allow_missing_func_imports,
)?;
let module = compiled_module.module_ref();
if !module.is_imported_memory(memory_index) {
// Inspect the module for the min and max memory sizes.
let (min_memory_size, max_memory_size) = {
let memory_plan = module.memory_plans
.get(memory_index)
.ok_or_else(|| WasmError::InvalidMemory)?;
(memory_plan.memory.minimum, memory_plan.memory.maximum)
};
let memory_plan = module.memory_plans.get(memory_index)
.expect("memory_index is retrieved from the module's exports map; qed");
(memory_plan.memory.minimum, memory_plan.memory.maximum)
};
// Check that heap_pages is within the allowed range.
let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size));
let heap_pages = heap_pages_valid(heap_pages, max_heap_pages)
.ok_or_else(|| WasmError::InvalidHeapPages)?;
// Check that heap_pages is within the allowed range.
let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size));
if max_heap_pages.map(|m| heap_pages > m).unwrap_or(false) {
return Err(WasmError::InvalidHeapPages)
}
}
Ok(WasmtimeRuntime {
module: compiled_module,
context,
max_heap_pages,
heap_pages,
host_functions,
memory_index,
})
}
@@ -209,8 +217,9 @@ fn scan_missing_functions(
fn create_compiled_unit(
code: &[u8],
host_functions: &[&'static dyn Function],
heap_pages: u32,
allow_missing_func_imports: bool,
) -> std::result::Result<(CompiledModule, Context), WasmError> {
) -> std::result::Result<(CompiledModule, Context, MemoryIndex), WasmError> {
let compilation_strategy = CompilationStrategy::Cranelift;
let compiler = new_compiler(compilation_strategy)?;
@@ -231,16 +240,32 @@ fn create_compiled_unit(
MissingFunctionStubs::new()
};
let env_missing_functions = missing_functions_stubs.stubs.remove("env").unwrap_or_else(|| Vec::new());
context.name_instance(
"env".to_owned(),
instantiate_env_module(global_exports, compiler, host_functions, env_missing_functions)?,
);
let env_missing_functions = missing_functions_stubs.stubs
.remove("env")
.unwrap_or_else(|| Vec::new());
let (module, memory_index) = instantiate_env_module(
global_exports,
compiler,
host_functions,
heap_pages,
env_missing_functions,
true,
)?;
context.name_instance("env".to_owned(), module);
for (module, missing_functions_stubs) in missing_functions_stubs.stubs {
let compiler = new_compiler(compilation_strategy)?;
let global_exports = context.get_global_exports();
let instance = instantiate_env_module(global_exports, compiler, &[], missing_functions_stubs)?;
let instance = instantiate_env_module(
global_exports,
compiler,
&[],
heap_pages,
missing_functions_stubs,
false,
)?.0;
context.name_instance(module, instance);
}
@@ -248,7 +273,7 @@ fn create_compiled_unit(
let module = context.compile_module(&code)
.map_err(|e| WasmError::Other(format!("module compile error: {}", e)))?;
Ok((module, context))
Ok((module, context, memory_index.expect("Memory is added on request; qed")))
}
/// Call a function inside a precompiled Wasm module.
@@ -257,22 +282,21 @@ fn call_method(
module: &mut CompiledModule,
method: &str,
data: &[u8],
memory_index: MemoryIndex,
heap_pages: u32,
) -> Result<Vec<u8>> {
let is_imported_memory = module.module().is_imported_memory(memory_index);
// Old exports get clobbered in `InstanceHandle::new` if we don't explicitly remove them first.
//
// The global exports mechanism is temporary in Wasmtime and expected to be removed.
// https://github.com/CraneStation/wasmtime/issues/332
clear_globals(&mut *context.get_global_exports().borrow_mut());
clear_globals(&mut *context.get_global_exports().borrow_mut(), is_imported_memory);
let mut instance = module.instantiate()
.map_err(|e| Error::Other(e.to_string()))?;
let mut instance = module.instantiate().map_err(|e| Error::Other(e.to_string()))?;
// Ideally there would be a way to set the heap pages during instantiation rather than
// growing the memory after the fact. Currently this may require an additional mmap and copy.
// However, the wasmtime API doesn't support modifying the size of memory on instantiation
// at this time.
grow_memory(&mut instance, heap_pages)?;
if !is_imported_memory {
grow_memory(&mut instance, heap_pages)?;
}
// Initialize the function executor state.
let heap_base = get_heap_base(&instance)?;
@@ -280,7 +304,7 @@ fn call_method(
reset_env_state_and_take_trap(context, Some(executor_state))?;
// Write the input data into guest memory.
let (data_ptr, data_len) = inject_input_data(context, &mut instance, data)?;
let (data_ptr, data_len) = inject_input_data(context, &mut instance, data, memory_index)?;
let args = [RuntimeValue::I32(u32::from(data_ptr) as i32), RuntimeValue::I32(data_len as i32)];
// Invoke the function in the runtime.
@@ -300,7 +324,7 @@ fn call_method(
// Read the output data from guest memory.
let mut output = vec![0; output_len as usize];
let memory = get_memory_mut(&mut instance)?;
let memory = get_memory_mut(&mut instance, memory_index)?;
read_memory_into(memory, Pointer::new(output_ptr), &mut output)?;
Ok(output)
}
@@ -310,9 +334,10 @@ fn instantiate_env_module(
global_exports: Rc<RefCell<HashMap<String, Option<Export>>>>,
compiler: Compiler,
host_functions: &[&'static dyn Function],
heap_pages: u32,
missing_functions_stubs: Vec<MissingFunction>,
) -> std::result::Result<InstanceHandle, WasmError>
{
add_memory: bool,
) -> std::result::Result<(InstanceHandle, Option<MemoryIndex>), WasmError> {
let isa = target_isa()?;
let pointer_type = isa.pointer_type();
let call_conv = isa.default_call_conv();
@@ -325,7 +350,7 @@ fn instantiate_env_module(
for function in host_functions {
let sig = translate_signature(
cranelift_ir_signature(function.signature(), &call_conv),
pointer_type
pointer_type,
);
let sig_id = module.signatures.push(sig.clone());
let func_id = module.functions.push(sig_id);
@@ -365,6 +390,22 @@ fn instantiate_env_module(
code_memory.publish();
let memory_id = if add_memory {
let memory = cranelift_wasm::Memory {
minimum: heap_pages + INITIAL_HEAP_PAGES,
maximum: Some(heap_pages + INITIAL_HEAP_PAGES),
shared: false,
};
let memory_plan = wasmtime_environ::MemoryPlan::for_memory(memory, &Default::default());
let memory_id = module.memory_plans.push(memory_plan);
module.exports.insert("memory".into(), wasmtime_environ::Export::Memory(memory_id));
Some(memory_id)
} else {
None
};
let imports = Imports::none();
let data_initializers = Vec::new();
let signatures = PrimaryMap::new();
@@ -380,7 +421,10 @@ fn instantiate_env_module(
None,
Box::new(env_state),
);
result.map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e)))
result
.map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e)))
.map(|r| (r, memory_id))
}
/// Build a new TargetIsa for the host machine.
@@ -396,8 +440,11 @@ fn new_compiler(strategy: CompilationStrategy) -> std::result::Result<Compiler,
Ok(Compiler::new(isa, strategy))
}
fn clear_globals(global_exports: &mut HashMap<String, Option<Export>>) {
global_exports.remove("memory");
fn clear_globals(global_exports: &mut HashMap<String, Option<Export>>, is_imported_memory: bool) {
// When memory is imported, we can not delete the global export.
if !is_imported_memory {
global_exports.remove("memory");
}
global_exports.remove("__heap_base");
global_exports.remove("__indirect_function_table");
}
@@ -441,13 +488,14 @@ fn inject_input_data(
context: &mut Context,
instance: &mut InstanceHandle,
data: &[u8],
memory_index: MemoryIndex,
) -> Result<(Pointer<u8>, WordSize)> {
let env_state = get_env_state(context)?;
let executor_state = env_state.executor_state
.as_mut()
.ok_or_else(|| "cannot get \"env\" module executor state")?;
let memory = get_memory_mut(instance)?;
let memory = get_memory_mut(instance, memory_index)?;
let data_len = data.len() as WordSize;
let data_ptr = executor_state.heap().allocate(memory, data_len)?;
@@ -455,12 +503,12 @@ fn inject_input_data(
Ok((data_ptr, data_len))
}
fn get_memory_mut(instance: &mut InstanceHandle) -> Result<&mut [u8]> {
match instance.lookup("memory") {
fn get_memory_mut(instance: &mut InstanceHandle, memory_index: MemoryIndex) -> Result<&mut [u8]> {
match instance.lookup_by_declaration(&wasmtime_environ::Export::Memory(memory_index)) {
// This is safe to wrap in an unsafe block as:
// - The definition pointer is returned by a lookup on a valid instance and thus points to
// a valid memory definition
Some(Export::Memory { definition, vmctx: _, memory: _ }) => unsafe {
Export::Memory { definition, vmctx: _, memory: _ } => unsafe {
Ok(std::slice::from_raw_parts_mut(
(*definition).base,
(*definition).current_length,
@@ -484,17 +532,3 @@ fn get_heap_base(instance: &InstanceHandle) -> Result<u32> {
}
}
}
/// Checks whether the heap_pages parameter is within the valid range and converts it to a u32.
/// Returns None if heaps_pages in not in range.
fn heap_pages_valid(heap_pages: u64, max_heap_pages: Option<u32>)
-> Option<u32>
{
let heap_pages = u32::try_from(heap_pages).ok()?;
if let Some(max_heap_pages) = max_heap_pages {
if heap_pages > max_heap_pages {
return None;
}
}
Some(heap_pages)
}
@@ -14,17 +14,13 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
use wasm_builder_runner::WasmBuilder;
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../../utils/wasm-builder",
version: "1.0.9",
},
// This instructs LLD to export __heap_base as a global variable, which is used by the
// external memory allocator.
"-Clink-arg=--export=__heap_base",
);
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder")
.export_heap_base()
.import_memory()
.build()
}
-1
View File
@@ -184,7 +184,6 @@ impl<Block: BlockT, Executor, Backend, G: GenesisInit> TestClientBuilder<Block,
Executor: sc_client::CallExecutor<Block> + 'static,
Backend: sc_client_api::backend::Backend<Block>,
{
let storage = {
let mut storage = self.genesis_init.genesis_storage();
+8 -13
View File
@@ -14,22 +14,17 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource};
use wasm_builder_runner::WasmBuilder;
fn main() {
build_current_project_with_rustflags(
"wasm_binary.rs",
WasmBuilderSource::CratesOrPath {
path: "../../utils/wasm-builder",
version: "1.0.9",
},
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_from_crates_or_path("1.0.9", "../../utils/wasm-builder")
.export_heap_base()
// Note that we set the stack-size to 1MB explicitly even though it is set
// to this value by default. This is because some of our tests (`restoration_of_globals`)
// depend on the stack-size.
//
// The --export=__heap_base instructs LLD to export __heap_base as a global variable, which
// is used by the external memory allocator.
"-Clink-arg=-zstack-size=1048576 \
-Clink-arg=--export=__heap_base",
);
.append_to_rust_flags("-Clink-arg=-zstack-size=1048576")
.import_memory()
.build()
}
-100
View File
@@ -255,8 +255,6 @@ cfg_if! {
fn use_trie() -> u64;
fn benchmark_indirect_call() -> u64;
fn benchmark_direct_call() -> u64;
fn returns_mutable_static() -> u64;
fn allocates_huge_stack_array(trap: bool) -> Vec<u8>;
fn vec_with_capacity(size: u32) -> Vec<u8>;
/// Returns the initialized block number.
fn get_block_number() -> u64;
@@ -299,8 +297,6 @@ cfg_if! {
fn use_trie() -> u64;
fn benchmark_indirect_call() -> u64;
fn benchmark_direct_call() -> u64;
fn returns_mutable_static() -> u64;
fn allocates_huge_stack_array(trap: bool) -> Vec<u8>;
fn vec_with_capacity(size: u32) -> Vec<u8>;
/// Returns the initialized block number.
fn get_block_number() -> u64;
@@ -448,11 +444,6 @@ impl_opaque_keys! {
}
}
#[cfg(not(feature = "std"))]
/// Mutable static variables should be always observed to have
/// the initialized value at the start of a runtime call.
static mut MUTABLE_STATIC: u64 = 32;
cfg_if! {
if #[cfg(feature = "std")] {
impl_runtime_apis! {
@@ -558,14 +549,6 @@ cfg_if! {
(0..1000).fold(0, |p, i| p + benchmark_add_one(i))
}
fn returns_mutable_static() -> u64 {
unimplemented!("is not expected to be invoked from non-wasm builds");
}
fn allocates_huge_stack_array(_trap: bool) -> Vec<u8> {
unimplemented!("is not expected to be invoked from non-wasm builds");
}
fn vec_with_capacity(_size: u32) -> Vec<u8> {
unimplemented!("is not expected to be invoked from non-wasm builds");
}
@@ -753,41 +736,6 @@ cfg_if! {
(0..10000).fold(0, |p, i| p + benchmark_add_one(i))
}
fn returns_mutable_static() -> u64 {
unsafe {
MUTABLE_STATIC += 1;
MUTABLE_STATIC
}
}
fn allocates_huge_stack_array(trap: bool) -> Vec<u8> {
// Allocate a stack frame that is approx. 75% of the stack (assuming it is 1MB).
// This will just decrease (stacks in wasm32-u-u grow downwards) the stack
// pointer. This won't trap on the current compilers.
let mut data = [0u8; 1024 * 768];
// Then make sure we actually write something to it.
//
// If:
// 1. the stack area is placed at the beginning of the linear memory space, and
// 2. the stack pointer points to out-of-bounds area, and
// 3. a write is performed around the current stack pointer.
//
// then a trap should happen.
//
for (i, v) in data.iter_mut().enumerate() {
*v = i as u8; // deliberate truncation
}
if trap {
// There is a small chance of this to be pulled up in theory. In practice
// the probability of that is rather low.
panic!()
}
data.to_vec()
}
fn vec_with_capacity(size: u32) -> Vec<u8> {
Vec::with_capacity(size as usize)
}
@@ -970,54 +918,6 @@ mod tests {
use sp_state_machine::ExecutionStrategy;
use codec::Encode;
#[test]
fn returns_mutable_static() {
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::AlwaysWasm)
.build();
let runtime_api = client.runtime_api();
let block_id = BlockId::Number(client.chain_info().best_number);
let ret = runtime_api.returns_mutable_static(&block_id).unwrap();
assert_eq!(ret, 33);
// We expect that every invocation will need to return the initial
// value plus one. If the value increases more than that then it is
// a sign that the wasm runtime preserves the memory content.
let ret = runtime_api.returns_mutable_static(&block_id).unwrap();
assert_eq!(ret, 33);
}
// If we didn't restore the wasm instance properly, on a trap the stack pointer would not be
// returned to its initial value and thus the stack space is going to be leaked.
//
// See https://github.com/paritytech/substrate/issues/2967 for details
#[test]
fn restoration_of_globals() {
// Allocate 32 pages (of 65536 bytes) which gives the runtime 2048KB of heap to operate on
// (plus some additional space unused from the initial pages requested by the wasm runtime
// module).
//
// The fixture performs 2 allocations of 768KB and this theoretically gives 1536KB, however, due
// to our allocator algorithm there are inefficiencies.
const REQUIRED_MEMORY_PAGES: u64 = 32;
let client = TestClientBuilder::new()
.set_execution_strategy(ExecutionStrategy::AlwaysWasm)
.set_heap_pages(REQUIRED_MEMORY_PAGES)
.build();
let runtime_api = client.runtime_api();
let block_id = BlockId::Number(client.chain_info().best_number);
// On the first invocation we allocate approx. 768KB (75%) of stack and then trap.
let ret = runtime_api.allocates_huge_stack_array(&block_id, true);
assert!(ret.is_err());
// On the second invocation we allocate yet another 768KB (75%) of stack
let ret = runtime_api.allocates_huge_stack_array(&block_id, false);
assert!(ret.is_ok());
}
#[test]
fn heap_pages_is_respected() {
// This tests that the on-chain HEAP_PAGES parameter is respected.
@@ -1,6 +1,6 @@
[package]
name = "substrate-wasm-builder-runner"
version = "1.0.4"
version = "1.0.5"
authors = ["Parity Technologies <admin@parity.io>"]
description = "Runner for substrate-wasm-builder"
edition = "2018"
+240 -39
View File
@@ -25,7 +25,10 @@
//!
//! For more information see <https://crates.io/substrate-wasm-builder>
use std::{env, process::{Command, self}, fs, path::{PathBuf, Path}};
use std::{
env, process::{Command, self}, fs, path::{PathBuf, Path}, hash::{Hash, Hasher},
collections::hash_map::DefaultHasher,
};
/// Environment variable that tells us to skip building the WASM binary.
const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
@@ -47,6 +50,225 @@ fn replace_back_slashes<T: ToString>(path: T) -> String {
path.to_string().replace("\\", "/")
}
/// Returns the manifest dir from the `CARGO_MANIFEST_DIR` env.
fn get_manifest_dir() -> PathBuf {
env::var("CARGO_MANIFEST_DIR")
.expect("`CARGO_MANIFEST_DIR` is always set for `build.rs` files; qed")
.into()
}
/// First step of the [`WasmBuilder`] to select the project to build.
pub struct WasmBuilderSelectProject {
/// This parameter just exists to make it impossible to construct
/// this type outside of this crate.
_ignore: (),
}
impl WasmBuilderSelectProject {
/// Use the current project as project for building the WASM binary.
///
/// # Panics
///
/// Panics if the `CARGO_MANIFEST_DIR` variable is not set. This variable
/// is always set by `Cargo` in `build.rs` files.
pub fn with_current_project(self) -> WasmBuilderSelectSource {
WasmBuilderSelectSource(get_manifest_dir().join("Cargo.toml"))
}
/// Use the given `path` as project for building the WASM binary.
///
/// Returns an error if the given `path` does not points to a `Cargo.toml`.
pub fn with_project(
self,
path: impl Into<PathBuf>,
) -> Result<WasmBuilderSelectSource, &'static str> {
let path = path.into();
if path.ends_with("Cargo.toml") {
Ok(WasmBuilderSelectSource(path))
} else {
Err("Project path must point to the `Cargo.toml` of the project")
}
}
}
/// Second step of the [`WasmBuilder`] to set the source of the `wasm-builder`.
pub struct WasmBuilderSelectSource(PathBuf);
impl WasmBuilderSelectSource {
/// Use the given `path` as source for `wasm-builder`.
///
/// The `path` must be relative and point to the directory that contains the `Cargo.toml` for
/// `wasm-builder`.
pub fn with_wasm_builder_from_path(self, path: &'static str) -> WasmBuilder {
WasmBuilder {
source: WasmBuilderSource::Path(path),
rust_flags: Vec::new(),
file_name: None,
project_cargo_toml: self.0,
}
}
/// Use the given `repo` and `rev` as source for `wasm-builder`.
pub fn with_wasm_builder_from_git(self, repo: &'static str, rev: &'static str) -> WasmBuilder {
WasmBuilder {
source: WasmBuilderSource::Git { repo, rev },
rust_flags: Vec::new(),
file_name: None,
project_cargo_toml: self.0,
}
}
/// Use the given `version` to fetch `wasm-builder` source from crates.io.
pub fn with_wasm_builder_from_crates(self, version: &'static str) -> WasmBuilder {
WasmBuilder {
source: WasmBuilderSource::Crates(version),
rust_flags: Vec::new(),
file_name: None,
project_cargo_toml: self.0,
}
}
/// Use the given `version` to fetch `wasm-builder` source from crates.io or use
/// the given `path` as source.
///
/// The `path` must be relative and point to the directory that contains the `Cargo.toml` for
/// `wasm-builder`.
pub fn with_wasm_builder_from_crates_or_path(
self,
version: &'static str,
path: &'static str,
) -> WasmBuilder {
WasmBuilder {
source: WasmBuilderSource::CratesOrPath { version, path },
rust_flags: Vec::new(),
file_name: None,
project_cargo_toml: self.0,
}
}
/// Use the given `source` as source for `wasm-builder`.
pub fn with_wasm_builder_source(self, source: WasmBuilderSource) -> WasmBuilder {
WasmBuilder {
source,
rust_flags: Vec::new(),
file_name: None,
project_cargo_toml: self.0,
}
}
}
/// The builder for building a wasm binary.
///
/// The builder itself is seperated into multiple structs to make the setup type safe.
///
/// Building a wasm binary:
///
/// 1. Call [`WasmBuilder::new`] to create a new builder.
/// 2. Select the project to build using the methods of [`WasmBuilderSelectProject`].
/// 3. Select the source of the `wasm-builder` crate using the methods of
/// [`WasmBuilderSelectSource`].
/// 4. Set additional `RUST_FLAGS` or a different name for the file containing the WASM code
/// using methods of [`Self`].
/// 5. Build the WASM binary using [`Self::build`].
pub struct WasmBuilder {
/// Where should we pull the `wasm-builder` crate from.
source: WasmBuilderSource,
/// Flags that should be appended to `RUST_FLAGS` env variable.
rust_flags: Vec<String>,
/// The name of the file that is being generated in `OUT_DIR`.
///
/// Defaults to `wasm_binary.rs`.
file_name: Option<String>,
/// The path to the `Cargo.toml` of the project that should be build
/// for wasm.
project_cargo_toml: PathBuf,
}
impl WasmBuilder {
/// Create a new instance of the builder.
pub fn new() -> WasmBuilderSelectProject {
WasmBuilderSelectProject {
_ignore: (),
}
}
/// Enable exporting `__heap_base` as global variable in the WASM binary.
///
/// This adds `-Clink-arg=--export=__heap_base` to `RUST_FLAGS`.
pub fn export_heap_base(mut self) -> Self {
self.rust_flags.push("-Clink-arg=--export=__heap_base".into());
self
}
/// Set the name of the file that will be generated in `OUT_DIR`.
///
/// This file needs to be included to get access to the build WASM binary.
///
/// If this function is not called, `file_name` defaults to `wasm_binary.rs`
pub fn set_file_name(mut self, file_name: impl Into<String>) -> Self {
self.file_name = Some(file_name.into());
self
}
/// Instruct the linker to import the memory into the WASM binary.
///
/// This adds `-C link-arg=--import-memory` to `RUST_FLAGS`.
pub fn import_memory(mut self) -> Self {
self.rust_flags.push("-C link-arg=--import-memory".into());
self
}
/// Append the given `flag` to `RUST_FLAGS`.
///
/// `flag` is appended as is, so it needs to be a valid flag.
pub fn append_to_rust_flags(mut self, flag: impl Into<String>) -> Self {
self.rust_flags.push(flag.into());
self
}
/// Build the WASM binary.
pub fn build(self) {
if check_skip_build() {
// If we skip the build, we still want to make sure to be called when an env variable
// changes
generate_rerun_if_changed_instructions();
return;
}
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!"));
let file_path = out_dir.join(self.file_name.unwrap_or_else(|| "wasm_binary.rs".into()));
// Hash the path to the project cargo toml.
let mut hasher = DefaultHasher::new();
self.project_cargo_toml.hash(&mut hasher);
let project_name = env::var("CARGO_PKG_NAME").expect("`CARGO_PKG_NAME` is set by cargo!");
// Make sure the `wasm-builder-runner` path is unique by concatenating the name of the
// project that is compiling the WASM binary with the hash of the path to the project that
// should be compiled as WASM binary.
let project_folder = get_workspace_root()
.join(format!("{}{}", project_name, hasher.finish()));
if check_provide_dummy_wasm_binary() {
provide_dummy_wasm_binary(&file_path);
} else {
create_project(
&project_folder,
&file_path,
self.source,
&self.project_cargo_toml,
&self.rust_flags.into_iter().map(|f| format!("{} ", f)).collect::<String>(),
);
run_project(&project_folder);
}
// As last step we need to generate our `rerun-if-changed` stuff. If a build fails, we don't
// want to spam the output!
generate_rerun_if_changed_instructions();
}
}
/// The `wasm-builder` dependency source.
pub enum WasmBuilderSource {
/// The relative path to the source code from the current manifest dir.
@@ -96,46 +318,21 @@ impl WasmBuilderSource {
/// Build the currently built project as WASM binary and extend `RUSTFLAGS` with the given rustflags.
///
/// For more information, see [`build_current_project`].
#[deprecated(
since = "1.0.5",
note = "Please switch to [`WasmBuilder`]",
)]
pub fn build_current_project_with_rustflags(
file_name: &str,
wasm_builder_source: WasmBuilderSource,
default_rustflags: &str,
default_rust_flags: &str,
) {
if check_skip_build() {
// If we skip the build, we still want to make sure to be called when an env variable changes
generate_rerun_if_changed_instructions();
return;
}
let manifest_dir = PathBuf::from(
env::var("CARGO_MANIFEST_DIR").expect(
"`CARGO_MANIFEST_DIR` is always set for `build.rs` files; qed"
)
);
let cargo_toml_path = manifest_dir.join("Cargo.toml");
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!"));
let file_path = out_dir.join(file_name);
let project_name = env::var("CARGO_PKG_NAME").expect("`CARGO_PKG_NAME` is set by cargo!");
let project_folder = get_workspace_root().join(project_name);
if check_provide_dummy_wasm_binary() {
provide_dummy_wasm_binary(&file_path);
} else {
create_project(
&project_folder,
&file_path,
&manifest_dir,
wasm_builder_source,
&cargo_toml_path,
default_rustflags,
);
run_project(&project_folder);
}
// As last step we need to generate our `rerun-if-changed` stuff. If a build fails, we don't
// want to spam the output!
generate_rerun_if_changed_instructions();
WasmBuilder::new()
.with_current_project()
.with_wasm_builder_source(wasm_builder_source)
.append_to_rust_flags(default_rust_flags)
.set_file_name(file_name)
.build()
}
/// Build the currently built project as WASM binary.
@@ -145,7 +342,12 @@ pub fn build_current_project_with_rustflags(
/// `file_name` - The name of the file being generated in the `OUT_DIR`. The file contains the
/// constant `WASM_BINARY` which contains the build wasm binary.
/// `wasm_builder_path` - Path to the wasm-builder project, relative to `CARGO_MANIFEST_DIR`.
#[deprecated(
since = "1.0.5",
note = "Please switch to [`WasmBuilder`]",
)]
pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSource) {
#[allow(deprecated)]
build_current_project_with_rustflags(file_name, wasm_builder_source, "");
}
@@ -171,7 +373,6 @@ fn get_workspace_root() -> PathBuf {
fn create_project(
project_folder: &Path,
file_path: &Path,
manifest_dir: &Path,
wasm_builder_source: WasmBuilderSource,
cargo_toml_path: &Path,
default_rustflags: &str,
@@ -193,7 +394,7 @@ fn create_project(
[workspace]
"#,
wasm_builder_source = wasm_builder_source.to_cargo_source(manifest_dir),
wasm_builder_source = wasm_builder_source.to_cargo_source(&get_manifest_dir()),
)
).expect("WASM build runner `Cargo.toml` writing can not fail; qed");