diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 4124e32263..ecf929384a 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -3234,7 +3234,7 @@ dependencies = [ "sp-std 2.0.0", "sp-transaction-pool 2.0.0", "sp-version 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "substrate-wasm-builder-runner 1.0.5", ] [[package]] @@ -3295,7 +3295,7 @@ dependencies = [ "sp-std 2.0.0", "sp-transaction-pool 2.0.0", "sp-version 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "substrate-wasm-builder-runner 1.0.5", ] [[package]] @@ -5823,7 +5823,7 @@ dependencies = [ "sp-runtime 2.0.0", "sp-sandbox 0.8.0", "sp-std 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "substrate-wasm-builder-runner 1.0.5", ] [[package]] @@ -6693,7 +6693,7 @@ dependencies = [ "sp-io 2.0.0", "sp-runtime-interface 2.0.0", "sp-std 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "substrate-wasm-builder-runner 1.0.5", ] [[package]] @@ -7063,7 +7063,7 @@ dependencies = [ "sp-trie 2.0.0", "sp-version 2.0.0", "substrate-test-runtime-client 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "substrate-wasm-builder-runner 1.0.5", "trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -7104,7 +7104,7 @@ dependencies = [ [[package]] name = "substrate-wasm-builder-runner" -version = "1.0.4" +version = "1.0.5" [[package]] name = "subtle" diff --git a/substrate/bin/node-template/runtime/build.rs b/substrate/bin/node-template/runtime/build.rs index 8bdf7584bb..39f7f56feb 100644 --- a/substrate/bin/node-template/runtime/build.rs +++ b/substrate/bin/node-template/runtime/build.rs @@ -1,11 +1,10 @@ -use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; +use wasm_builder_runner::WasmBuilder; fn main() { - build_current_project_with_rustflags( - "wasm_binary.rs", - WasmBuilderSource::Crates("1.0.9"), - // This instructs LLD to export __heap_base as a global variable, which is used by the - // external memory allocator. - "-Clink-arg=--export=__heap_base", - ); + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_from_crates("1.0.9") + .export_heap_base() + .import_memory() + .build() } diff --git a/substrate/bin/node/runtime/build.rs b/substrate/bin/node/runtime/build.rs index 9c81ea6f38..647b476814 100644 --- a/substrate/bin/node/runtime/build.rs +++ b/substrate/bin/node/runtime/build.rs @@ -14,17 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; +use wasm_builder_runner::WasmBuilder; fn main() { - build_current_project_with_rustflags( - "wasm_binary.rs", - WasmBuilderSource::CratesOrPath { - path: "../../../utils/wasm-builder", - version: "1.0.9", - }, - // This instructs LLD to export __heap_base as a global variable, which is used by the - // external memory allocator. - "-Clink-arg=--export=__heap_base", - ); + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .export_heap_base() + .import_memory() + .build() } diff --git a/substrate/client/executor/runtime-test/build.rs b/substrate/client/executor/runtime-test/build.rs index 9c81ea6f38..647b476814 100644 --- a/substrate/client/executor/runtime-test/build.rs +++ b/substrate/client/executor/runtime-test/build.rs @@ -14,17 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; +use wasm_builder_runner::WasmBuilder; fn main() { - build_current_project_with_rustflags( - "wasm_binary.rs", - WasmBuilderSource::CratesOrPath { - path: "../../../utils/wasm-builder", - version: "1.0.9", - }, - // This instructs LLD to export __heap_base as a global variable, which is used by the - // external memory allocator. - "-Clink-arg=--export=__heap_base", - ); + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .export_heap_base() + .import_memory() + .build() } diff --git a/substrate/client/executor/runtime-test/src/lib.rs b/substrate/client/executor/runtime-test/src/lib.rs index c0807197b4..f6d37abf94 100644 --- a/substrate/client/executor/runtime-test/src/lib.rs +++ b/substrate/client/executor/runtime-test/src/lib.rs @@ -26,6 +26,11 @@ extern "C" { fn yet_another_missing_external(); } +#[cfg(not(feature = "std"))] +/// Mutable static variables should be always observed to have +/// the initialized value at the start of a runtime call. +static mut MUTABLE_STATIC: u64 = 32; + sp_core::wasm_export_functions! { fn test_calling_missing_external() { unsafe { missing_external() } @@ -217,6 +222,41 @@ sp_core::wasm_export_functions! { fn test_sp_allocator_compiles() { sp_allocator::FreeingBumpHeapAllocator::new(0); } + + fn returns_mutable_static() -> u64 { + unsafe { + MUTABLE_STATIC += 1; + MUTABLE_STATIC + } + } + + fn allocates_huge_stack_array(trap: bool) -> Vec { + // Allocate a stack frame that is approx. 75% of the stack (assuming it is 1MB). + // This will just decrease (stacks in wasm32-u-u grow downwards) the stack + // pointer. This won't trap on the current compilers. + let mut data = [0u8; 1024 * 768]; + + // Then make sure we actually write something to it. + // + // If: + // 1. the stack area is placed at the beginning of the linear memory space, and + // 2. the stack pointer points to out-of-bounds area, and + // 3. a write is performed around the current stack pointer. + // + // then a trap should happen. + // + for (i, v) in data.iter_mut().enumerate() { + *v = i as u8; // deliberate truncation + } + + if trap { + // There is a small chance of this to be pulled up in theory. In practice + // the probability of that is rather low. + panic!() + } + + data.to_vec() + } } #[cfg(not(feature = "std"))] diff --git a/substrate/client/executor/src/integration_tests/mod.rs b/substrate/client/executor/src/integration_tests/mod.rs index 5b2fc5ca0c..ef8171e5ad 100644 --- a/substrate/client/executor/src/integration_tests/mod.rs +++ b/substrate/client/executor/src/integration_tests/mod.rs @@ -27,6 +27,7 @@ use sc_runtime_test::WASM_BINARY; use sp_state_machine::TestExternalities as CoreTestExternalities; use test_case::test_case; use sp_trie::{TrieConfiguration, trie_types::Layout}; +use sp_wasm_interface::HostFunctions as _; use crate::WasmExecutionMethod; @@ -38,16 +39,14 @@ fn call_in_wasm( call_data: &[u8], execution_method: WasmExecutionMethod, ext: &mut E, - code: &[u8], - heap_pages: u64, ) -> crate::error::Result> { crate::call_in_wasm::( function, call_data, execution_method, ext, - code, - heap_pages, + &WASM_BINARY[..], + 1024, true, ) } @@ -57,15 +56,12 @@ fn call_in_wasm( fn returning_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let output = call_in_wasm( "test_empty_return", &[], wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(); assert_eq!(output, vec![0u8; 0]); } @@ -73,17 +69,14 @@ fn returning_should_work(wasm_method: WasmExecutionMethod) { #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn call_not_existing_function(wasm_method: WasmExecutionMethod) { - let mut ext = TestExternalities::default(); - let mut ext = ext.ext(); - let test_code = WASM_BINARY; + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); match call_in_wasm( "test_calling_missing_external", &[], wasm_method, &mut ext, - &test_code[..], - 8, ) { Ok(_) => panic!("was expected an `Err`"), Err(e) => { @@ -107,15 +100,12 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; match call_in_wasm( "test_calling_yet_another_missing_external", &[], wasm_method, &mut ext, - &test_code[..], - 8, ) { Ok(_) => panic!("was expected an `Err`"), Err(e) => { @@ -139,15 +129,12 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { fn panicking_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let output = call_in_wasm( "test_panic", &[], wasm_method, &mut ext, - &test_code[..], - 8, ); assert!(output.is_err()); @@ -156,8 +143,6 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) { &[0], wasm_method, &mut ext, - &test_code[..], - 8, ); assert_eq!(Decode::decode(&mut &output.unwrap()[..]), Ok(Vec::::new())); @@ -166,8 +151,6 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) { &vec![2].encode(), wasm_method, &mut ext, - &test_code[..], - 8, ); assert!(output.is_err()); } @@ -180,15 +163,12 @@ fn storage_should_work(wasm_method: WasmExecutionMethod) { { let mut ext = ext.ext(); ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); - let test_code = WASM_BINARY; let output = call_in_wasm( "test_data_in", &b"Hello world".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); @@ -216,7 +196,6 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { ext.set_storage(b"aba".to_vec(), b"3".to_vec()); ext.set_storage(b"abb".to_vec(), b"4".to_vec()); ext.set_storage(b"bbb".to_vec(), b"5".to_vec()); - let test_code = WASM_BINARY; // This will clear all entries which prefix is "ab". let output = call_in_wasm( @@ -224,8 +203,6 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { &b"ab".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); @@ -247,15 +224,12 @@ fn clear_prefix_should_work(wasm_method: WasmExecutionMethod) { fn blake2_256_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_blake2_256", &[0], wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), blake2_256(&b""[..]).to_vec().encode(), ); @@ -265,8 +239,6 @@ fn blake2_256_should_work(wasm_method: WasmExecutionMethod) { &b"Hello world!".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), blake2_256(&b"Hello world!"[..]).to_vec().encode(), ); @@ -277,15 +249,12 @@ fn blake2_256_should_work(wasm_method: WasmExecutionMethod) { fn blake2_128_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_blake2_128", &[0], wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), blake2_128(&b""[..]).to_vec().encode(), ); @@ -295,8 +264,6 @@ fn blake2_128_should_work(wasm_method: WasmExecutionMethod) { &b"Hello world!".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), blake2_128(&b"Hello world!"[..]).to_vec().encode(), ); @@ -307,15 +274,12 @@ fn blake2_128_should_work(wasm_method: WasmExecutionMethod) { fn sha2_256_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_sha2_256", &[0], wasm_method, &mut ext, - &test_code[..], - 8, ) .unwrap(), hex!("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") @@ -328,8 +292,6 @@ fn sha2_256_should_work(wasm_method: WasmExecutionMethod) { &b"Hello world!".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ) .unwrap(), hex!("c0535e4be2b79ffd93291305436bf889314e4a3faec05ecffcbb7df31ad9e51a") @@ -343,15 +305,12 @@ fn sha2_256_should_work(wasm_method: WasmExecutionMethod) { fn twox_256_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_twox_256", &[0], wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), hex!( "99e9d85137db46ef4bbea33613baafd56f963c64b1f3685a4eb4abd67ff6203a" @@ -363,8 +322,6 @@ fn twox_256_should_work(wasm_method: WasmExecutionMethod) { &b"Hello world!".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), hex!( "b27dfd7f223f177f2a13647b533599af0c07f68bda23d96d059da2b451a35a74" @@ -377,15 +334,12 @@ fn twox_256_should_work(wasm_method: WasmExecutionMethod) { fn twox_128_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_twox_128", &[0], wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(), ); @@ -395,8 +349,6 @@ fn twox_128_should_work(wasm_method: WasmExecutionMethod) { &b"Hello world!".to_vec().encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), hex!("b27dfd7f223f177f2a13647b533599af").to_vec().encode(), ); @@ -407,7 +359,6 @@ fn twox_128_should_work(wasm_method: WasmExecutionMethod) { fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let key = ed25519::Pair::from_seed(&blake2_256(b"test")); let sig = key.sign(b"all ok!"); let mut calldata = vec![]; @@ -420,8 +371,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) { &calldata.encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -437,8 +386,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) { &calldata.encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), false.encode(), ); @@ -449,7 +396,6 @@ fn ed25519_verify_should_work(wasm_method: WasmExecutionMethod) { fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let key = sr25519::Pair::from_seed(&blake2_256(b"test")); let sig = key.sign(b"all ok!"); let mut calldata = vec![]; @@ -462,8 +408,6 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) { &calldata.encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -479,8 +423,6 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) { &calldata.encode(), wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), false.encode(), ); @@ -490,17 +432,13 @@ fn sr25519_verify_should_work(wasm_method: WasmExecutionMethod) { #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn ordered_trie_root_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); - let mut ext = ext.ext(); let trie_input = vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]; - let test_code = WASM_BINARY; assert_eq!( call_in_wasm( "test_ordered_trie_root", &[0], wasm_method, - &mut ext, - &test_code[..], - 8, + &mut ext.ext(), ).unwrap(), Layout::::ordered_trie_root(trie_input.iter()).as_bytes().encode(), ); @@ -514,16 +452,12 @@ fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); ext.register_extension(OffchainExt::new(offchain)); - let test_code = WASM_BINARY; - let mut ext = ext.ext(); assert_eq!( call_in_wasm( "test_offchain_local_storage", &[0], wasm_method, - &mut ext, - &test_code[..], - 8, + &mut ext.ext(), ).unwrap(), true.encode(), ); @@ -550,17 +484,84 @@ fn offchain_http_should_work(wasm_method: WasmExecutionMethod) { }, ); - let test_code = WASM_BINARY; - let mut ext = ext.ext(); assert_eq!( call_in_wasm( "test_offchain_http", &[0], wasm_method, - &mut ext, - &test_code[..], - 8, + &mut ext.ext(), ).unwrap(), true.encode(), ); } + +#[test_case(WasmExecutionMethod::Interpreted)] +#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] +#[should_panic(expected = "Allocator ran out of space")] +fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + + crate::call_in_wasm::( + "test_exhaust_heap", + &[0], + wasm_method, + &mut ext.ext(), + &WASM_BINARY[..], + // `17` is the initial number of pages compiled into the binary. + 17, + true, + ).unwrap(); +} + +#[test_case(WasmExecutionMethod::Interpreted)] +#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] +fn returns_mutable_static(wasm_method: WasmExecutionMethod) { + let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + wasm_method, + 1024, + &WASM_BINARY[..], + HostFunctions::host_functions(), + true, + ).expect("Creates instance"); + + let res = instance.call("returns_mutable_static", &[0]).unwrap(); + assert_eq!(33, u64::decode(&mut &res[..]).unwrap()); + + // We expect that every invocation will need to return the initial + // value plus one. If the value increases more than that then it is + // a sign that the wasm runtime preserves the memory content. + let res = instance.call("returns_mutable_static", &[0]).unwrap(); + assert_eq!(33, u64::decode(&mut &res[..]).unwrap()); +} + +// If we didn't restore the wasm instance properly, on a trap the stack pointer would not be +// returned to its initial value and thus the stack space is going to be leaked. +// +// See https://github.com/paritytech/substrate/issues/2967 for details +#[test_case(WasmExecutionMethod::Interpreted)] +#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] +fn restoration_of_globals(wasm_method: WasmExecutionMethod) { + // Allocate 32 pages (of 65536 bytes) which gives the runtime 2048KB of heap to operate on + // (plus some additional space unused from the initial pages requested by the wasm runtime + // module). + // + // The fixture performs 2 allocations of 768KB and this theoretically gives 1536KB, however, due + // to our allocator algorithm there are inefficiencies. + const REQUIRED_MEMORY_PAGES: u64 = 32; + + let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + wasm_method, + REQUIRED_MEMORY_PAGES, + &WASM_BINARY[..], + HostFunctions::host_functions(), + true, + ).expect("Creates instance"); + + // On the first invocation we allocate approx. 768KB (75%) of stack and then trap. + let res = instance.call("allocates_huge_stack_array", &true.encode()); + assert!(res.is_err()); + + // On the second invocation we allocate yet another 768KB (75%) of stack + let res = instance.call("allocates_huge_stack_array", &false.encode()); + assert!(res.is_ok()); +} diff --git a/substrate/client/executor/src/integration_tests/sandbox.rs b/substrate/client/executor/src/integration_tests/sandbox.rs index 9a9b33608d..9458542e3a 100644 --- a/substrate/client/executor/src/integration_tests/sandbox.rs +++ b/substrate/client/executor/src/integration_tests/sandbox.rs @@ -18,7 +18,6 @@ use super::{TestExternalities, call_in_wasm}; use crate::WasmExecutionMethod; use codec::Encode; -use sc_runtime_test::WASM_BINARY; use test_case::test_case; use wabt; @@ -27,7 +26,6 @@ use wabt; fn sandbox_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -56,8 +54,6 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -68,7 +64,6 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) { fn sandbox_trap(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -86,47 +81,16 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), vec![0], ); } -#[test_case(WasmExecutionMethod::Interpreted)] -#[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] -#[should_panic(expected = "Allocator ran out of space")] -fn sandbox_should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { - let mut ext = TestExternalities::default(); - let mut ext = ext.ext(); - let test_code = WASM_BINARY; - - let code = wabt::wat2wasm(r#" - (module - (import "env" "assert" (func $assert (param i32))) - (func (export "call") - i32.const 0 - call $assert - ) - ) - "#).unwrap().encode(); - - call_in_wasm( - "test_exhaust_heap", - &code, - wasm_method, - &mut ext, - &test_code[..], - 8, - ).unwrap(); -} - #[test_case(WasmExecutionMethod::Interpreted)] #[cfg_attr(feature = "wasmtime", test_case(WasmExecutionMethod::Compiled))] fn start_called(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -161,8 +125,6 @@ fn start_called(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -173,7 +135,6 @@ fn start_called(wasm_method: WasmExecutionMethod) { fn invoke_args(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -204,8 +165,6 @@ fn invoke_args(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -216,7 +175,6 @@ fn invoke_args(wasm_method: WasmExecutionMethod) { fn return_val(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -235,8 +193,6 @@ fn return_val(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), true.encode(), ); @@ -247,7 +203,6 @@ fn return_val(wasm_method: WasmExecutionMethod) { fn unlinkable_module(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -264,8 +219,6 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), 1u8.encode(), ); @@ -276,7 +229,6 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) { fn corrupted_module(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; // Corrupted wasm file let code = vec![0u8, 0, 0, 0, 1, 0, 0, 0].encode(); @@ -287,8 +239,6 @@ fn corrupted_module(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), 1u8.encode(), ); @@ -299,7 +249,6 @@ fn corrupted_module(wasm_method: WasmExecutionMethod) { fn start_fn_ok(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -319,8 +268,6 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), 0u8.encode(), ); @@ -331,7 +278,6 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) { fn start_fn_traps(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let test_code = WASM_BINARY; let code = wabt::wat2wasm(r#" (module @@ -352,8 +298,6 @@ fn start_fn_traps(wasm_method: WasmExecutionMethod) { &code, wasm_method, &mut ext, - &test_code[..], - 8, ).unwrap(), 2u8.encode(), ); diff --git a/substrate/client/executor/wasmi/src/lib.rs b/substrate/client/executor/wasmi/src/lib.rs index cf8125e774..c4701fd827 100644 --- a/substrate/client/executor/wasmi/src/lib.rs +++ b/substrate/client/executor/wasmi/src/lib.rs @@ -24,7 +24,7 @@ use wasmi::{ }; use codec::{Encode, Decode}; use sp_core::sandbox as sandbox_primitives; -use log::{error, trace}; +use log::{error, trace, debug}; use parity_wasm::elements::{deserialize_buffer, DataSegment, Instruction, Module as RawModule}; use sp_wasm_interface::{ FunctionContext, Pointer, WordSize, Sandbox, MemoryId, Result as WResult, Function, @@ -271,18 +271,36 @@ impl<'a> Sandbox for FunctionExecutor<'a> { } } +/// Will be used on initialization of a module to resolve function and memory imports. struct Resolver<'a> { - host_functions: &'a[&'static dyn Function], + /// All the hot functions that we export for the WASM blob. + host_functions: &'a [&'static dyn Function], + /// Should we allow missing function imports? + /// + /// If `true`, we return a stub that will return an error when being called. allow_missing_func_imports: bool, + /// All the names of functions for that we did not provide a host function. missing_functions: RefCell>, + /// Will be used as initial and maximum size of the imported memory. + heap_pages: usize, + /// By default, runtimes should import memory and this is `Some(_)` after + /// reolving. However, to be backwards compatible, we also support memory + /// exported by the WASM blob (this will be `None` after resolving). + import_memory: RefCell>, } impl<'a> Resolver<'a> { - fn new(host_functions: &'a[&'static dyn Function], allow_missing_func_imports: bool) -> Resolver<'a> { + fn new( + host_functions: &'a[&'static dyn Function], + allow_missing_func_imports: bool, + heap_pages: usize, + ) -> Resolver<'a> { Resolver { host_functions, allow_missing_func_imports, missing_functions: RefCell::new(Vec::new()), + heap_pages, + import_memory: Default::default(), } } } @@ -323,6 +341,48 @@ impl<'a> wasmi::ModuleImportResolver for Resolver<'a> { )) } } + + fn resolve_memory( + &self, + field_name: &str, + memory_type: &wasmi::MemoryDescriptor, + ) -> Result { + if field_name == "memory" { + match &mut *self.import_memory.borrow_mut() { + Some(_) => Err(wasmi::Error::Instantiation( + "Memory can not be imported twice!".into(), + )), + memory_ref @ None => { + if memory_type + .maximum() + .map(|m| m.saturating_sub(memory_type.initial())) + .map(|m| self.heap_pages > m as usize) + .unwrap_or(false) + { + Err(wasmi::Error::Instantiation(format!( + "Heap pages ({}) is greater than imported memory maximum ({}).", + self.heap_pages, + memory_type + .maximum() + .map(|m| m.saturating_sub(memory_type.initial())) + .expect("Maximum is set, checked above; qed"), + ))) + } else { + let memory = MemoryInstance::alloc( + Pages(memory_type.initial() as usize + self.heap_pages), + Some(Pages(memory_type.initial() as usize + self.heap_pages)), + )?; + *memory_ref = Some(memory.clone()); + Ok(memory) + } + } + } + } else { + Err(wasmi::Error::Instantiation( + format!("Unknown memory reference with name: {}", field_name), + )) + } + } } impl<'a> wasmi::Externals for FunctionExecutor<'a> { @@ -378,15 +438,14 @@ fn get_heap_base(module: &ModuleRef) -> Result { /// Call a given method in the given wasm-module runtime. fn call_in_wasm_module( module_instance: &ModuleRef, + memory: &MemoryRef, method: &str, data: &[u8], host_functions: &[&'static dyn Function], allow_missing_func_imports: bool, missing_functions: &Vec, ) -> Result, Error> { - // extract a reference to a linear memory, optional reference to a table - // and then initialize FunctionExecutor. - let memory = get_mem_instance(module_instance)?; + // Initialize FunctionExecutor. let table: Option = module_instance .export_by_name("__indirect_function_table") .and_then(|e| e.as_table().cloned()); @@ -434,8 +493,8 @@ fn instantiate_module( module: &Module, host_functions: &[&'static dyn Function], allow_missing_func_imports: bool, -) -> Result<(ModuleRef, Vec), Error> { - let resolver = Resolver::new(host_functions, allow_missing_func_imports); +) -> Result<(ModuleRef, Vec, MemoryRef), Error> { + let resolver = Resolver::new(host_functions, allow_missing_func_imports, heap_pages); // start module instantiation. Don't run 'start' function yet. let intermediate_instance = ModuleInstance::new( module, @@ -445,15 +504,33 @@ fn instantiate_module( // Verify that the module has the heap base global variable. let _ = get_heap_base(intermediate_instance.not_started_instance())?; - // Extract a reference to a linear memory. - let memory = get_mem_instance(intermediate_instance.not_started_instance())?; - memory.grow(Pages(heap_pages)).map_err(|_| Error::Runtime)?; + + // Get the memory reference. Runtimes should import memory, but to be backwards + // compatible we also support exported memory. + let memory = match resolver.import_memory.into_inner() { + Some(memory) => memory, + None => { + debug!( + target: "wasm-executor", + "WASM blob does not imports memory, falling back to exported memory", + ); + + let memory = get_mem_instance(intermediate_instance.not_started_instance())?; + memory.grow(Pages(heap_pages)).map_err(|_| Error::Runtime)?; + + memory + } + }; if intermediate_instance.has_start() { // Runtime is not allowed to have the `start` function. Err(Error::RuntimeHasStartFn) } else { - Ok((intermediate_instance.assert_no_start(), resolver.missing_functions.into_inner())) + Ok(( + intermediate_instance.assert_no_start(), + resolver.missing_functions.into_inner(), + memory, + )) } } @@ -528,14 +605,7 @@ impl StateSnapshot { /// the preserved memory and globals. /// /// Returns `Err` if applying the snapshot is failed. - fn apply(&self, instance: &ModuleRef) -> Result<(), WasmError> { - let memory = instance - .export_by_name("memory") - .ok_or(WasmError::ApplySnapshotFailed)? - .as_memory() - .cloned() - .ok_or(WasmError::ApplySnapshotFailed)?; - + fn apply(&self, instance: &ModuleRef, memory: &MemoryRef) -> Result<(), WasmError> { // First, erase the memory and copy the data segments into it. memory .erase() @@ -569,6 +639,8 @@ impl StateSnapshot { pub struct WasmiRuntime { /// A wasm module instance. instance: ModuleRef, + /// The memory instance of used by the wasm module. + memory: MemoryRef, /// The snapshot of the instance's state taken just after the instantiation. state_snapshot: StateSnapshot, /// The host functions registered for this instance. @@ -594,7 +666,7 @@ impl WasmRuntime for WasmiRuntime { method: &str, data: &[u8], ) -> Result, Error> { - self.state_snapshot.apply(&self.instance) + self.state_snapshot.apply(&self.instance, &self.memory) .map_err(|e| { // Snapshot restoration failed. This is pretty unexpected since this can happen // if some invariant is broken or if the system is under extreme memory pressure @@ -604,6 +676,7 @@ impl WasmRuntime for WasmiRuntime { })?; call_in_wasm_module( &self.instance, + &self.memory, method, data, &self.host_functions, @@ -628,7 +701,7 @@ pub fn create_instance( let data_segments = extract_data_segments(&code)?; // Instantiate this module. - let (instance, missing_functions) = instantiate_module( + let (instance, missing_functions, memory) = instantiate_module( heap_pages as usize, &module, &host_functions, @@ -646,6 +719,7 @@ pub fn create_instance( Ok(WasmiRuntime { instance, + memory, state_snapshot, host_functions, allow_missing_func_imports, diff --git a/substrate/client/executor/wasmtime/src/runtime.rs b/substrate/client/executor/wasmtime/src/runtime.rs index 77a11ddc7d..abf860667f 100644 --- a/substrate/client/executor/wasmtime/src/runtime.rs +++ b/substrate/client/executor/wasmtime/src/runtime.rs @@ -32,42 +32,38 @@ use sc_executor_common::{ use sp_wasm_interface::{Pointer, WordSize, Function}; use sp_runtime_interface::unpack_ptr_and_len; -use std::cell::RefCell; -use std::collections::HashMap; -use std::convert::TryFrom; -use std::rc::Rc; +use std::{cell::RefCell, collections::HashMap, convert::TryFrom, rc::Rc}; use cranelift_codegen::ir; use cranelift_codegen::isa::TargetIsa; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_frontend::FunctionBuilderContext; -use cranelift_wasm::DefinedFuncIndex; +use cranelift_wasm::{DefinedFuncIndex, MemoryIndex}; use wasmtime_environ::{Module, translate_signature}; use wasmtime_jit::{ ActionOutcome, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, RuntimeValue, }; use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody}; +/// TODO: We should remove this in https://github.com/paritytech/substrate/pull/4686 +/// Currently there is no way to extract this with wasmtime. +const INITIAL_HEAP_PAGES: u32 = 17; + /// A `WasmRuntime` implementation using the Wasmtime JIT to compile the runtime module to native /// and execute the compiled code. pub struct WasmtimeRuntime { module: CompiledModule, context: Context, - max_heap_pages: Option, heap_pages: u32, /// The host functions registered for this instance. host_functions: Vec<&'static dyn Function>, + /// The index of the memory in the module. + memory_index: MemoryIndex, } impl WasmRuntime for WasmtimeRuntime { fn update_heap_pages(&mut self, heap_pages: u64) -> bool { - match heap_pages_valid(heap_pages, self.max_heap_pages) { - Some(heap_pages) => { - self.heap_pages = heap_pages; - true - } - None => false, - } + self.heap_pages as u64 == heap_pages } fn host_functions(&self) -> &[&'static dyn Function] { @@ -80,6 +76,7 @@ impl WasmRuntime for WasmtimeRuntime { &mut self.module, method, data, + self.memory_index, self.heap_pages, ) } @@ -93,31 +90,42 @@ pub fn create_instance( host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, ) -> std::result::Result { - let (compiled_module, context) = create_compiled_unit(code, &host_functions, allow_missing_func_imports)?; + let heap_pages = u32::try_from(heap_pages) + .map_err(|e| + WasmError::Other(format!("Heap pages can not be converted into `u32`: {:?}", e)) + )?; - // Inspect the module for the min and max memory sizes. - let (min_memory_size, max_memory_size) = { - let module = compiled_module.module_ref(); - let memory_index = match module.exports.get("memory") { - Some(wasmtime_environ::Export::Memory(memory_index)) => *memory_index, - _ => return Err(WasmError::InvalidMemory), + let (compiled_module, context, memory_index) = create_compiled_unit( + code, + &host_functions, + heap_pages, + allow_missing_func_imports, + )?; + + let module = compiled_module.module_ref(); + if !module.is_imported_memory(memory_index) { + // Inspect the module for the min and max memory sizes. + let (min_memory_size, max_memory_size) = { + let memory_plan = module.memory_plans + .get(memory_index) + .ok_or_else(|| WasmError::InvalidMemory)?; + (memory_plan.memory.minimum, memory_plan.memory.maximum) }; - let memory_plan = module.memory_plans.get(memory_index) - .expect("memory_index is retrieved from the module's exports map; qed"); - (memory_plan.memory.minimum, memory_plan.memory.maximum) - }; - // Check that heap_pages is within the allowed range. - let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size)); - let heap_pages = heap_pages_valid(heap_pages, max_heap_pages) - .ok_or_else(|| WasmError::InvalidHeapPages)?; + // Check that heap_pages is within the allowed range. + let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size)); + + if max_heap_pages.map(|m| heap_pages > m).unwrap_or(false) { + return Err(WasmError::InvalidHeapPages) + } + } Ok(WasmtimeRuntime { module: compiled_module, context, - max_heap_pages, heap_pages, host_functions, + memory_index, }) } @@ -209,8 +217,9 @@ fn scan_missing_functions( fn create_compiled_unit( code: &[u8], host_functions: &[&'static dyn Function], + heap_pages: u32, allow_missing_func_imports: bool, -) -> std::result::Result<(CompiledModule, Context), WasmError> { +) -> std::result::Result<(CompiledModule, Context, MemoryIndex), WasmError> { let compilation_strategy = CompilationStrategy::Cranelift; let compiler = new_compiler(compilation_strategy)?; @@ -231,16 +240,32 @@ fn create_compiled_unit( MissingFunctionStubs::new() }; - let env_missing_functions = missing_functions_stubs.stubs.remove("env").unwrap_or_else(|| Vec::new()); - context.name_instance( - "env".to_owned(), - instantiate_env_module(global_exports, compiler, host_functions, env_missing_functions)?, - ); + let env_missing_functions = missing_functions_stubs.stubs + .remove("env") + .unwrap_or_else(|| Vec::new()); + + let (module, memory_index) = instantiate_env_module( + global_exports, + compiler, + host_functions, + heap_pages, + env_missing_functions, + true, + )?; + + context.name_instance("env".to_owned(), module); for (module, missing_functions_stubs) in missing_functions_stubs.stubs { let compiler = new_compiler(compilation_strategy)?; let global_exports = context.get_global_exports(); - let instance = instantiate_env_module(global_exports, compiler, &[], missing_functions_stubs)?; + let instance = instantiate_env_module( + global_exports, + compiler, + &[], + heap_pages, + missing_functions_stubs, + false, + )?.0; context.name_instance(module, instance); } @@ -248,7 +273,7 @@ fn create_compiled_unit( let module = context.compile_module(&code) .map_err(|e| WasmError::Other(format!("module compile error: {}", e)))?; - Ok((module, context)) + Ok((module, context, memory_index.expect("Memory is added on request; qed"))) } /// Call a function inside a precompiled Wasm module. @@ -257,22 +282,21 @@ fn call_method( module: &mut CompiledModule, method: &str, data: &[u8], + memory_index: MemoryIndex, heap_pages: u32, ) -> Result> { + let is_imported_memory = module.module().is_imported_memory(memory_index); // Old exports get clobbered in `InstanceHandle::new` if we don't explicitly remove them first. // // The global exports mechanism is temporary in Wasmtime and expected to be removed. // https://github.com/CraneStation/wasmtime/issues/332 - clear_globals(&mut *context.get_global_exports().borrow_mut()); + clear_globals(&mut *context.get_global_exports().borrow_mut(), is_imported_memory); - let mut instance = module.instantiate() - .map_err(|e| Error::Other(e.to_string()))?; + let mut instance = module.instantiate().map_err(|e| Error::Other(e.to_string()))?; - // Ideally there would be a way to set the heap pages during instantiation rather than - // growing the memory after the fact. Currently this may require an additional mmap and copy. - // However, the wasmtime API doesn't support modifying the size of memory on instantiation - // at this time. - grow_memory(&mut instance, heap_pages)?; + if !is_imported_memory { + grow_memory(&mut instance, heap_pages)?; + } // Initialize the function executor state. let heap_base = get_heap_base(&instance)?; @@ -280,7 +304,7 @@ fn call_method( reset_env_state_and_take_trap(context, Some(executor_state))?; // Write the input data into guest memory. - let (data_ptr, data_len) = inject_input_data(context, &mut instance, data)?; + let (data_ptr, data_len) = inject_input_data(context, &mut instance, data, memory_index)?; let args = [RuntimeValue::I32(u32::from(data_ptr) as i32), RuntimeValue::I32(data_len as i32)]; // Invoke the function in the runtime. @@ -300,7 +324,7 @@ fn call_method( // Read the output data from guest memory. let mut output = vec![0; output_len as usize]; - let memory = get_memory_mut(&mut instance)?; + let memory = get_memory_mut(&mut instance, memory_index)?; read_memory_into(memory, Pointer::new(output_ptr), &mut output)?; Ok(output) } @@ -310,9 +334,10 @@ fn instantiate_env_module( global_exports: Rc>>>, compiler: Compiler, host_functions: &[&'static dyn Function], + heap_pages: u32, missing_functions_stubs: Vec, -) -> std::result::Result -{ + add_memory: bool, +) -> std::result::Result<(InstanceHandle, Option), WasmError> { let isa = target_isa()?; let pointer_type = isa.pointer_type(); let call_conv = isa.default_call_conv(); @@ -325,7 +350,7 @@ fn instantiate_env_module( for function in host_functions { let sig = translate_signature( cranelift_ir_signature(function.signature(), &call_conv), - pointer_type + pointer_type, ); let sig_id = module.signatures.push(sig.clone()); let func_id = module.functions.push(sig_id); @@ -365,6 +390,22 @@ fn instantiate_env_module( code_memory.publish(); + let memory_id = if add_memory { + let memory = cranelift_wasm::Memory { + minimum: heap_pages + INITIAL_HEAP_PAGES, + maximum: Some(heap_pages + INITIAL_HEAP_PAGES), + shared: false, + }; + let memory_plan = wasmtime_environ::MemoryPlan::for_memory(memory, &Default::default()); + + let memory_id = module.memory_plans.push(memory_plan); + module.exports.insert("memory".into(), wasmtime_environ::Export::Memory(memory_id)); + + Some(memory_id) + } else { + None + }; + let imports = Imports::none(); let data_initializers = Vec::new(); let signatures = PrimaryMap::new(); @@ -380,7 +421,10 @@ fn instantiate_env_module( None, Box::new(env_state), ); - result.map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e))) + + result + .map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e))) + .map(|r| (r, memory_id)) } /// Build a new TargetIsa for the host machine. @@ -396,8 +440,11 @@ fn new_compiler(strategy: CompilationStrategy) -> std::result::Result>) { - global_exports.remove("memory"); +fn clear_globals(global_exports: &mut HashMap>, is_imported_memory: bool) { + // When memory is imported, we can not delete the global export. + if !is_imported_memory { + global_exports.remove("memory"); + } global_exports.remove("__heap_base"); global_exports.remove("__indirect_function_table"); } @@ -441,13 +488,14 @@ fn inject_input_data( context: &mut Context, instance: &mut InstanceHandle, data: &[u8], + memory_index: MemoryIndex, ) -> Result<(Pointer, WordSize)> { let env_state = get_env_state(context)?; let executor_state = env_state.executor_state .as_mut() .ok_or_else(|| "cannot get \"env\" module executor state")?; - let memory = get_memory_mut(instance)?; + let memory = get_memory_mut(instance, memory_index)?; let data_len = data.len() as WordSize; let data_ptr = executor_state.heap().allocate(memory, data_len)?; @@ -455,12 +503,12 @@ fn inject_input_data( Ok((data_ptr, data_len)) } -fn get_memory_mut(instance: &mut InstanceHandle) -> Result<&mut [u8]> { - match instance.lookup("memory") { +fn get_memory_mut(instance: &mut InstanceHandle, memory_index: MemoryIndex) -> Result<&mut [u8]> { + match instance.lookup_by_declaration(&wasmtime_environ::Export::Memory(memory_index)) { // This is safe to wrap in an unsafe block as: // - The definition pointer is returned by a lookup on a valid instance and thus points to // a valid memory definition - Some(Export::Memory { definition, vmctx: _, memory: _ }) => unsafe { + Export::Memory { definition, vmctx: _, memory: _ } => unsafe { Ok(std::slice::from_raw_parts_mut( (*definition).base, (*definition).current_length, @@ -484,17 +532,3 @@ fn get_heap_base(instance: &InstanceHandle) -> Result { } } } - -/// Checks whether the heap_pages parameter is within the valid range and converts it to a u32. -/// Returns None if heaps_pages in not in range. -fn heap_pages_valid(heap_pages: u64, max_heap_pages: Option) - -> Option -{ - let heap_pages = u32::try_from(heap_pages).ok()?; - if let Some(max_heap_pages) = max_heap_pages { - if heap_pages > max_heap_pages { - return None; - } - } - Some(heap_pages) -} diff --git a/substrate/primitives/runtime-interface/test-wasm/build.rs b/substrate/primitives/runtime-interface/test-wasm/build.rs index 9c81ea6f38..647b476814 100644 --- a/substrate/primitives/runtime-interface/test-wasm/build.rs +++ b/substrate/primitives/runtime-interface/test-wasm/build.rs @@ -14,17 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; +use wasm_builder_runner::WasmBuilder; fn main() { - build_current_project_with_rustflags( - "wasm_binary.rs", - WasmBuilderSource::CratesOrPath { - path: "../../../utils/wasm-builder", - version: "1.0.9", - }, - // This instructs LLD to export __heap_base as a global variable, which is used by the - // external memory allocator. - "-Clink-arg=--export=__heap_base", - ); + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_from_crates_or_path("1.0.9", "../../../utils/wasm-builder") + .export_heap_base() + .import_memory() + .build() } diff --git a/substrate/test-utils/client/src/lib.rs b/substrate/test-utils/client/src/lib.rs index 76aa1acd8a..e95c5ad162 100644 --- a/substrate/test-utils/client/src/lib.rs +++ b/substrate/test-utils/client/src/lib.rs @@ -184,7 +184,6 @@ impl TestClientBuilder + 'static, Backend: sc_client_api::backend::Backend, { - let storage = { let mut storage = self.genesis_init.genesis_storage(); diff --git a/substrate/test-utils/runtime/build.rs b/substrate/test-utils/runtime/build.rs index 8a9ee642c4..1fd3d52b2f 100644 --- a/substrate/test-utils/runtime/build.rs +++ b/substrate/test-utils/runtime/build.rs @@ -14,22 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use wasm_builder_runner::{build_current_project_with_rustflags, WasmBuilderSource}; +use wasm_builder_runner::WasmBuilder; fn main() { - build_current_project_with_rustflags( - "wasm_binary.rs", - WasmBuilderSource::CratesOrPath { - path: "../../utils/wasm-builder", - version: "1.0.9", - }, + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_from_crates_or_path("1.0.9", "../../utils/wasm-builder") + .export_heap_base() // Note that we set the stack-size to 1MB explicitly even though it is set // to this value by default. This is because some of our tests (`restoration_of_globals`) // depend on the stack-size. - // - // The --export=__heap_base instructs LLD to export __heap_base as a global variable, which - // is used by the external memory allocator. - "-Clink-arg=-zstack-size=1048576 \ - -Clink-arg=--export=__heap_base", - ); + .append_to_rust_flags("-Clink-arg=-zstack-size=1048576") + .import_memory() + .build() } diff --git a/substrate/test-utils/runtime/src/lib.rs b/substrate/test-utils/runtime/src/lib.rs index b8e858c3a4..96387b1efc 100644 --- a/substrate/test-utils/runtime/src/lib.rs +++ b/substrate/test-utils/runtime/src/lib.rs @@ -255,8 +255,6 @@ cfg_if! { fn use_trie() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; - fn returns_mutable_static() -> u64; - fn allocates_huge_stack_array(trap: bool) -> Vec; fn vec_with_capacity(size: u32) -> Vec; /// Returns the initialized block number. fn get_block_number() -> u64; @@ -299,8 +297,6 @@ cfg_if! { fn use_trie() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; - fn returns_mutable_static() -> u64; - fn allocates_huge_stack_array(trap: bool) -> Vec; fn vec_with_capacity(size: u32) -> Vec; /// Returns the initialized block number. fn get_block_number() -> u64; @@ -448,11 +444,6 @@ impl_opaque_keys! { } } -#[cfg(not(feature = "std"))] -/// Mutable static variables should be always observed to have -/// the initialized value at the start of a runtime call. -static mut MUTABLE_STATIC: u64 = 32; - cfg_if! { if #[cfg(feature = "std")] { impl_runtime_apis! { @@ -558,14 +549,6 @@ cfg_if! { (0..1000).fold(0, |p, i| p + benchmark_add_one(i)) } - fn returns_mutable_static() -> u64 { - unimplemented!("is not expected to be invoked from non-wasm builds"); - } - - fn allocates_huge_stack_array(_trap: bool) -> Vec { - unimplemented!("is not expected to be invoked from non-wasm builds"); - } - fn vec_with_capacity(_size: u32) -> Vec { unimplemented!("is not expected to be invoked from non-wasm builds"); } @@ -753,41 +736,6 @@ cfg_if! { (0..10000).fold(0, |p, i| p + benchmark_add_one(i)) } - fn returns_mutable_static() -> u64 { - unsafe { - MUTABLE_STATIC += 1; - MUTABLE_STATIC - } - } - - fn allocates_huge_stack_array(trap: bool) -> Vec { - // Allocate a stack frame that is approx. 75% of the stack (assuming it is 1MB). - // This will just decrease (stacks in wasm32-u-u grow downwards) the stack - // pointer. This won't trap on the current compilers. - let mut data = [0u8; 1024 * 768]; - - // Then make sure we actually write something to it. - // - // If: - // 1. the stack area is placed at the beginning of the linear memory space, and - // 2. the stack pointer points to out-of-bounds area, and - // 3. a write is performed around the current stack pointer. - // - // then a trap should happen. - // - for (i, v) in data.iter_mut().enumerate() { - *v = i as u8; // deliberate truncation - } - - if trap { - // There is a small chance of this to be pulled up in theory. In practice - // the probability of that is rather low. - panic!() - } - - data.to_vec() - } - fn vec_with_capacity(size: u32) -> Vec { Vec::with_capacity(size as usize) } @@ -970,54 +918,6 @@ mod tests { use sp_state_machine::ExecutionStrategy; use codec::Encode; - #[test] - fn returns_mutable_static() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); - let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.chain_info().best_number); - - let ret = runtime_api.returns_mutable_static(&block_id).unwrap(); - assert_eq!(ret, 33); - - // We expect that every invocation will need to return the initial - // value plus one. If the value increases more than that then it is - // a sign that the wasm runtime preserves the memory content. - let ret = runtime_api.returns_mutable_static(&block_id).unwrap(); - assert_eq!(ret, 33); - } - - // If we didn't restore the wasm instance properly, on a trap the stack pointer would not be - // returned to its initial value and thus the stack space is going to be leaked. - // - // See https://github.com/paritytech/substrate/issues/2967 for details - #[test] - fn restoration_of_globals() { - // Allocate 32 pages (of 65536 bytes) which gives the runtime 2048KB of heap to operate on - // (plus some additional space unused from the initial pages requested by the wasm runtime - // module). - // - // The fixture performs 2 allocations of 768KB and this theoretically gives 1536KB, however, due - // to our allocator algorithm there are inefficiencies. - const REQUIRED_MEMORY_PAGES: u64 = 32; - - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .set_heap_pages(REQUIRED_MEMORY_PAGES) - .build(); - let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.chain_info().best_number); - - // On the first invocation we allocate approx. 768KB (75%) of stack and then trap. - let ret = runtime_api.allocates_huge_stack_array(&block_id, true); - assert!(ret.is_err()); - - // On the second invocation we allocate yet another 768KB (75%) of stack - let ret = runtime_api.allocates_huge_stack_array(&block_id, false); - assert!(ret.is_ok()); - } - #[test] fn heap_pages_is_respected() { // This tests that the on-chain HEAP_PAGES parameter is respected. diff --git a/substrate/utils/wasm-builder-runner/Cargo.toml b/substrate/utils/wasm-builder-runner/Cargo.toml index ab8a539054..1380d64fb3 100644 --- a/substrate/utils/wasm-builder-runner/Cargo.toml +++ b/substrate/utils/wasm-builder-runner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "substrate-wasm-builder-runner" -version = "1.0.4" +version = "1.0.5" authors = ["Parity Technologies "] description = "Runner for substrate-wasm-builder" edition = "2018" diff --git a/substrate/utils/wasm-builder-runner/src/lib.rs b/substrate/utils/wasm-builder-runner/src/lib.rs index 40016a0159..6366c1ce37 100644 --- a/substrate/utils/wasm-builder-runner/src/lib.rs +++ b/substrate/utils/wasm-builder-runner/src/lib.rs @@ -25,7 +25,10 @@ //! //! For more information see -use std::{env, process::{Command, self}, fs, path::{PathBuf, Path}}; +use std::{ + env, process::{Command, self}, fs, path::{PathBuf, Path}, hash::{Hash, Hasher}, + collections::hash_map::DefaultHasher, +}; /// Environment variable that tells us to skip building the WASM binary. const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD"; @@ -47,6 +50,225 @@ fn replace_back_slashes(path: T) -> String { path.to_string().replace("\\", "/") } +/// Returns the manifest dir from the `CARGO_MANIFEST_DIR` env. +fn get_manifest_dir() -> PathBuf { + env::var("CARGO_MANIFEST_DIR") + .expect("`CARGO_MANIFEST_DIR` is always set for `build.rs` files; qed") + .into() +} + +/// First step of the [`WasmBuilder`] to select the project to build. +pub struct WasmBuilderSelectProject { + /// This parameter just exists to make it impossible to construct + /// this type outside of this crate. + _ignore: (), +} + +impl WasmBuilderSelectProject { + /// Use the current project as project for building the WASM binary. + /// + /// # Panics + /// + /// Panics if the `CARGO_MANIFEST_DIR` variable is not set. This variable + /// is always set by `Cargo` in `build.rs` files. + pub fn with_current_project(self) -> WasmBuilderSelectSource { + WasmBuilderSelectSource(get_manifest_dir().join("Cargo.toml")) + } + + /// Use the given `path` as project for building the WASM binary. + /// + /// Returns an error if the given `path` does not points to a `Cargo.toml`. + pub fn with_project( + self, + path: impl Into, + ) -> Result { + let path = path.into(); + + if path.ends_with("Cargo.toml") { + Ok(WasmBuilderSelectSource(path)) + } else { + Err("Project path must point to the `Cargo.toml` of the project") + } + } +} + +/// Second step of the [`WasmBuilder`] to set the source of the `wasm-builder`. +pub struct WasmBuilderSelectSource(PathBuf); + +impl WasmBuilderSelectSource { + /// Use the given `path` as source for `wasm-builder`. + /// + /// The `path` must be relative and point to the directory that contains the `Cargo.toml` for + /// `wasm-builder`. + pub fn with_wasm_builder_from_path(self, path: &'static str) -> WasmBuilder { + WasmBuilder { + source: WasmBuilderSource::Path(path), + rust_flags: Vec::new(), + file_name: None, + project_cargo_toml: self.0, + } + } + + /// Use the given `repo` and `rev` as source for `wasm-builder`. + pub fn with_wasm_builder_from_git(self, repo: &'static str, rev: &'static str) -> WasmBuilder { + WasmBuilder { + source: WasmBuilderSource::Git { repo, rev }, + rust_flags: Vec::new(), + file_name: None, + project_cargo_toml: self.0, + } + } + + /// Use the given `version` to fetch `wasm-builder` source from crates.io. + pub fn with_wasm_builder_from_crates(self, version: &'static str) -> WasmBuilder { + WasmBuilder { + source: WasmBuilderSource::Crates(version), + rust_flags: Vec::new(), + file_name: None, + project_cargo_toml: self.0, + } + } + + /// Use the given `version` to fetch `wasm-builder` source from crates.io or use + /// the given `path` as source. + /// + /// The `path` must be relative and point to the directory that contains the `Cargo.toml` for + /// `wasm-builder`. + pub fn with_wasm_builder_from_crates_or_path( + self, + version: &'static str, + path: &'static str, + ) -> WasmBuilder { + WasmBuilder { + source: WasmBuilderSource::CratesOrPath { version, path }, + rust_flags: Vec::new(), + file_name: None, + project_cargo_toml: self.0, + } + } + + /// Use the given `source` as source for `wasm-builder`. + pub fn with_wasm_builder_source(self, source: WasmBuilderSource) -> WasmBuilder { + WasmBuilder { + source, + rust_flags: Vec::new(), + file_name: None, + project_cargo_toml: self.0, + } + } +} + +/// The builder for building a wasm binary. +/// +/// The builder itself is seperated into multiple structs to make the setup type safe. +/// +/// Building a wasm binary: +/// +/// 1. Call [`WasmBuilder::new`] to create a new builder. +/// 2. Select the project to build using the methods of [`WasmBuilderSelectProject`]. +/// 3. Select the source of the `wasm-builder` crate using the methods of +/// [`WasmBuilderSelectSource`]. +/// 4. Set additional `RUST_FLAGS` or a different name for the file containing the WASM code +/// using methods of [`Self`]. +/// 5. Build the WASM binary using [`Self::build`]. +pub struct WasmBuilder { + /// Where should we pull the `wasm-builder` crate from. + source: WasmBuilderSource, + /// Flags that should be appended to `RUST_FLAGS` env variable. + rust_flags: Vec, + /// The name of the file that is being generated in `OUT_DIR`. + /// + /// Defaults to `wasm_binary.rs`. + file_name: Option, + /// The path to the `Cargo.toml` of the project that should be build + /// for wasm. + project_cargo_toml: PathBuf, +} + +impl WasmBuilder { + /// Create a new instance of the builder. + pub fn new() -> WasmBuilderSelectProject { + WasmBuilderSelectProject { + _ignore: (), + } + } + + /// Enable exporting `__heap_base` as global variable in the WASM binary. + /// + /// This adds `-Clink-arg=--export=__heap_base` to `RUST_FLAGS`. + pub fn export_heap_base(mut self) -> Self { + self.rust_flags.push("-Clink-arg=--export=__heap_base".into()); + self + } + + /// Set the name of the file that will be generated in `OUT_DIR`. + /// + /// This file needs to be included to get access to the build WASM binary. + /// + /// If this function is not called, `file_name` defaults to `wasm_binary.rs` + pub fn set_file_name(mut self, file_name: impl Into) -> Self { + self.file_name = Some(file_name.into()); + self + } + + /// Instruct the linker to import the memory into the WASM binary. + /// + /// This adds `-C link-arg=--import-memory` to `RUST_FLAGS`. + pub fn import_memory(mut self) -> Self { + self.rust_flags.push("-C link-arg=--import-memory".into()); + self + } + + /// Append the given `flag` to `RUST_FLAGS`. + /// + /// `flag` is appended as is, so it needs to be a valid flag. + pub fn append_to_rust_flags(mut self, flag: impl Into) -> Self { + self.rust_flags.push(flag.into()); + self + } + + /// Build the WASM binary. + pub fn build(self) { + if check_skip_build() { + // If we skip the build, we still want to make sure to be called when an env variable + // changes + generate_rerun_if_changed_instructions(); + return; + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!")); + let file_path = out_dir.join(self.file_name.unwrap_or_else(|| "wasm_binary.rs".into())); + + // Hash the path to the project cargo toml. + let mut hasher = DefaultHasher::new(); + self.project_cargo_toml.hash(&mut hasher); + + let project_name = env::var("CARGO_PKG_NAME").expect("`CARGO_PKG_NAME` is set by cargo!"); + // Make sure the `wasm-builder-runner` path is unique by concatenating the name of the + // project that is compiling the WASM binary with the hash of the path to the project that + // should be compiled as WASM binary. + let project_folder = get_workspace_root() + .join(format!("{}{}", project_name, hasher.finish())); + + if check_provide_dummy_wasm_binary() { + provide_dummy_wasm_binary(&file_path); + } else { + create_project( + &project_folder, + &file_path, + self.source, + &self.project_cargo_toml, + &self.rust_flags.into_iter().map(|f| format!("{} ", f)).collect::(), + ); + run_project(&project_folder); + } + + // As last step we need to generate our `rerun-if-changed` stuff. If a build fails, we don't + // want to spam the output! + generate_rerun_if_changed_instructions(); + } +} + /// The `wasm-builder` dependency source. pub enum WasmBuilderSource { /// The relative path to the source code from the current manifest dir. @@ -96,46 +318,21 @@ impl WasmBuilderSource { /// Build the currently built project as WASM binary and extend `RUSTFLAGS` with the given rustflags. /// /// For more information, see [`build_current_project`]. +#[deprecated( + since = "1.0.5", + note = "Please switch to [`WasmBuilder`]", +)] pub fn build_current_project_with_rustflags( file_name: &str, wasm_builder_source: WasmBuilderSource, - default_rustflags: &str, + default_rust_flags: &str, ) { - if check_skip_build() { - // If we skip the build, we still want to make sure to be called when an env variable changes - generate_rerun_if_changed_instructions(); - return; - } - - let manifest_dir = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect( - "`CARGO_MANIFEST_DIR` is always set for `build.rs` files; qed" - ) - ); - - let cargo_toml_path = manifest_dir.join("Cargo.toml"); - let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!")); - let file_path = out_dir.join(file_name); - let project_name = env::var("CARGO_PKG_NAME").expect("`CARGO_PKG_NAME` is set by cargo!"); - let project_folder = get_workspace_root().join(project_name); - - if check_provide_dummy_wasm_binary() { - provide_dummy_wasm_binary(&file_path); - } else { - create_project( - &project_folder, - &file_path, - &manifest_dir, - wasm_builder_source, - &cargo_toml_path, - default_rustflags, - ); - run_project(&project_folder); - } - - // As last step we need to generate our `rerun-if-changed` stuff. If a build fails, we don't - // want to spam the output! - generate_rerun_if_changed_instructions(); + WasmBuilder::new() + .with_current_project() + .with_wasm_builder_source(wasm_builder_source) + .append_to_rust_flags(default_rust_flags) + .set_file_name(file_name) + .build() } /// Build the currently built project as WASM binary. @@ -145,7 +342,12 @@ pub fn build_current_project_with_rustflags( /// `file_name` - The name of the file being generated in the `OUT_DIR`. The file contains the /// constant `WASM_BINARY` which contains the build wasm binary. /// `wasm_builder_path` - Path to the wasm-builder project, relative to `CARGO_MANIFEST_DIR`. +#[deprecated( + since = "1.0.5", + note = "Please switch to [`WasmBuilder`]", +)] pub fn build_current_project(file_name: &str, wasm_builder_source: WasmBuilderSource) { + #[allow(deprecated)] build_current_project_with_rustflags(file_name, wasm_builder_source, ""); } @@ -171,7 +373,6 @@ fn get_workspace_root() -> PathBuf { fn create_project( project_folder: &Path, file_path: &Path, - manifest_dir: &Path, wasm_builder_source: WasmBuilderSource, cargo_toml_path: &Path, default_rustflags: &str, @@ -193,7 +394,7 @@ fn create_project( [workspace] "#, - wasm_builder_source = wasm_builder_source.to_cargo_source(manifest_dir), + wasm_builder_source = wasm_builder_source.to_cargo_source(&get_manifest_dir()), ) ).expect("WASM build runner `Cargo.toml` writing can not fail; qed");