pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-04-26 21:37:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

wasm-executor: Support growing the memory (#12520)

Browse Source 
* As always, start with something :P

* Add support for max_heap_pages

* Add support for wasmtime

* Make it compile

* Fix compilation

* Copy wrongly merged code

* Fix compilation

* Some fixes

* Fix

* Get stuff working

* More work

* More fixes

* ...

* More

* FIXEs

* Switch wasmi to use `RuntimeBlob` like wasmtime

* Removed unused stuff

* Cleanup

* More cleanups

* Introduce `CallContext`

* Fixes

* More fixes

* Add builder for creating the `WasmExecutor`

* Adds some docs

* FMT

* First round of feedback.

* Review feedback round 2

* More fixes

* Fix try-runtime

* Update client/executor/wasmtime/src/instance_wrapper.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/executor/common/src/wasm_runtime.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/executor/common/src/runtime_blob/runtime_blob.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/executor/common/src/wasm_runtime.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/allocator/src/freeing_bump.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Update client/allocator/src/freeing_bump.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Feedback round 3

* FMT

* Review comments

---------

Co-authored-by: Koute <koute@users.noreply.github.com>
This commit is contained in:
Bastian Köcher
2023-02-24 12:43:01 +01:00
committed by GitHub
parent c848d40775
commit 941288c6d0
37 changed files with 1092 additions and 667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/executor/wasmtime/src/runtime.rs
+22 -62
View File
@@ -20,7 +20,7 @@
use crate::{
host::HostState,
instance_wrapper::{EntryPoint, InstanceWrapper},
instance_wrapper::{EntryPoint, InstanceWrapper, MemoryWrapper},
util::{self, replace_strategy_if_broken},
};
@@ -31,7 +31,7 @@ use sc_executor_common::{
self, DataSegmentsSnapshot, ExposedMutableGlobalsSet, GlobalsSnapshot, RuntimeBlob,
},
util::checked_range,
wasm_runtime::{InvokeMethod, WasmInstance, WasmModule},
wasm_runtime::{HeapAllocStrategy, InvokeMethod, WasmInstance, WasmModule},
};
use sp_runtime_interface::unpack_ptr_and_len;
use sp_wasm_interface::{HostFunctions, Pointer, Value, WordSize};
@@ -42,12 +42,10 @@ use std::{
Arc,
},
};
use wasmtime::{AsContext, Engine, Memory, StoreLimits, Table};
use wasmtime::{AsContext, Engine, Memory, Table};
#[derive(Default)]
pub(crate) struct StoreData {
/// The limits we apply to the store. We need to store it here to return a reference to this
/// object when we have the limits enabled.
pub(crate) limits: StoreLimits,
/// This will only be set when we call into the runtime.
pub(crate) host_state: Option<HostState>,
/// This will be always set once the store is initialized.
@@ -83,12 +81,11 @@ enum Strategy {
struct InstanceCreator {
engine: wasmtime::Engine,
instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
max_memory_size: Option<usize>,
}
impl InstanceCreator {
fn instantiate(&mut self) -> Result<InstanceWrapper> {
InstanceWrapper::new(&self.engine, &self.instance_pre, self.max_memory_size)
InstanceWrapper::new(&self.engine, &self.instance_pre)
}
}
@@ -128,18 +125,13 @@ pub struct WasmtimeRuntime {
engine: wasmtime::Engine,
instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
instantiation_strategy: InternalInstantiationStrategy,
config: Config,
}
impl WasmModule for WasmtimeRuntime {
fn new_instance(&self) -> Result<Box<dyn WasmInstance>> {
let strategy = match self.instantiation_strategy {
InternalInstantiationStrategy::LegacyInstanceReuse(ref snapshot_data) => {
let mut instance_wrapper = InstanceWrapper::new(
&self.engine,
&self.instance_pre,
self.config.semantics.max_memory_size,
)?;
let mut instance_wrapper = InstanceWrapper::new(&self.engine, &self.instance_pre)?;
let heap_base = instance_wrapper.extract_heap_base()?;
// This function panics if the instance was created from a runtime blob different
@@ -161,7 +153,6 @@ impl WasmModule for WasmtimeRuntime {
InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator {
engine: self.engine.clone(),
instance_pre: self.instance_pre.clone(),
max_memory_size: self.config.semantics.max_memory_size,
}),
};
@@ -350,25 +341,22 @@ fn common_config(semantics: &Semantics) -> std::result::Result<wasmtime::Config,
InstantiationStrategy::LegacyInstanceReuse => (false, false),
};
const WASM_PAGE_SIZE: u64 = 65536;
config.memory_init_cow(use_cow);
config.memory_guaranteed_dense_image_size(
semantics.max_memory_size.map(|max| max as u64).unwrap_or(u64::MAX),
);
config.memory_guaranteed_dense_image_size(match semantics.heap_alloc_strategy {
HeapAllocStrategy::Dynamic { maximum_pages } =>
maximum_pages.map(|p| p as u64 * WASM_PAGE_SIZE).unwrap_or(u64::MAX),
HeapAllocStrategy::Static { .. } => u64::MAX,
});
if use_pooling {
const WASM_PAGE_SIZE: u64 = 65536;
const MAX_WASM_PAGES: u64 = 0x10000;
let memory_pages = if let Some(max_memory_size) = semantics.max_memory_size {
let max_memory_size = max_memory_size as u64;
let mut pages = max_memory_size / WASM_PAGE_SIZE;
if max_memory_size % WASM_PAGE_SIZE != 0 {
pages += 1;
}
std::cmp::min(MAX_WASM_PAGES, pages)
} else {
MAX_WASM_PAGES
let memory_pages = match semantics.heap_alloc_strategy {
HeapAllocStrategy::Dynamic { maximum_pages } =>
maximum_pages.map(|p| p as u64).unwrap_or(MAX_WASM_PAGES),
HeapAllocStrategy::Static { .. } => MAX_WASM_PAGES,
};
let mut pooling_config = wasmtime::PoolingAllocationConfig::default();
@@ -514,25 +502,8 @@ pub struct Semantics {
/// Configures wasmtime to use multiple threads for compiling.
pub parallel_compilation: bool,
/// The number of extra WASM pages which will be allocated
/// on top of what is requested by the WASM blob itself.
pub extra_heap_pages: u64,
/// The total amount of memory in bytes an instance can request.
///
/// If specified, the runtime will be able to allocate only that much of wasm memory.
/// This is the total number and therefore the [`Semantics::extra_heap_pages`] is accounted
/// for.
///
/// That means that the initial number of pages of a linear memory plus the
/// [`Semantics::extra_heap_pages`] multiplied by the wasm page size (64KiB) should be less
/// than or equal to `max_memory_size`, otherwise the instance won't be created.
///
/// Moreover, `memory.grow` will fail (return -1) if the sum of sizes of currently mounted
/// and additional pages exceeds `max_memory_size`.
///
/// The default is `None`.
pub max_memory_size: Option<usize>,
/// The heap allocation strategy to use.
pub heap_alloc_strategy: HeapAllocStrategy,
}
#[derive(Clone)]
@@ -689,12 +660,7 @@ where
.instantiate_pre(&module)
.map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?;
Ok(WasmtimeRuntime {
engine,
instance_pre: Arc::new(instance_pre),
instantiation_strategy,
config,
})
Ok(WasmtimeRuntime { engine, instance_pre: Arc::new(instance_pre), instantiation_strategy })
}
fn prepare_blob_for_compilation(
@@ -717,12 +683,7 @@ fn prepare_blob_for_compilation(
// now automatically take care of creating the memory for us, and it is also necessary
// to enable `wasmtime`'s instance pooling. (Imported memories are ineligible for pooling.)
blob.convert_memory_import_into_export()?;
blob.add_extra_heap_pages_to_memory_section(
semantics
.extra_heap_pages
.try_into()
.map_err(|e| WasmError::Other(format!("invalid `extra_heap_pages`: {}", e)))?,
)?;
blob.setup_memory_according_to_heap_alloc_strategy(semantics.heap_alloc_strategy)?;
Ok(blob)
}
@@ -783,9 +744,8 @@ fn inject_input_data(
) -> Result<(Pointer<u8>, WordSize)> {
let mut ctx = instance.store_mut();
let memory = ctx.data().memory();
let memory = memory.data_mut(&mut ctx);
let data_len = data.len() as WordSize;
let data_ptr = allocator.allocate(memory, data_len)?;
let data_ptr = allocator.allocate(&mut MemoryWrapper(&memory, &mut ctx), data_len)?;
util::write_memory_from(instance.store_mut(), data_ptr, data)?;
Ok((data_ptr, data_len))
}