Embed runtime version as a custom section (#8688)

* emit a custom section from impl_runtime_apis!

This change emits a custom section from the impl_runtime_apis! proc macro.

Each implemented API will result to emitting a link section `runtime_apis`.
During linking all sections with this name will be concatenated and
placed into the final wasm binary under the same name.

* Introduce `runtime_version` proc macro

This macro takes an existing `RuntimeVersion` const declaration, parses
it and emits the version information in form of a linking section.
Ultimately such a linking section will result into a custom wasm
section.

* Parse custom wasm section for runtime version

* Apply suggestions from code review

Co-authored-by: David <dvdplm@gmail.com>

* Fix sc-executor integration tests

* Nits

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Refactor apis section deserialization

* Fix version decoding

* Reuse uncompressed value for CallInWasm

* Log on decompression error

* Simplify if

* Reexport proc-macro from sp_version

* Merge ReadRuntimeVersionExt

* Export `read_embedded_version`

* Fix test

* Simplify searching for custom section

Co-authored-by: David <dvdplm@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
This commit is contained in:
Sergei Shulepov
2021-05-12 15:39:08 +02:00
committed by GitHub
parent 4f7c1df31e
commit 0849bcce0e
27 changed files with 833 additions and 289 deletions
@@ -17,18 +17,20 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod sandbox;
use std::sync::Arc;
use codec::{Encode, Decode};
use hex_literal::hex;
use sp_core::{
blake2_128, blake2_256, ed25519, sr25519, map, Pair,
offchain::{OffchainWorkerExt, OffchainDbExt, testing},
traits::{Externalities, CallInWasm},
traits::Externalities,
};
use sc_runtime_test::wasm_binary_unwrap;
use sp_state_machine::TestExternalities as CoreTestExternalities;
use sp_trie::{TrieConfiguration, trie_types::Layout};
use sp_wasm_interface::HostFunctions as _;
use sp_runtime::traits::BlakeTwo256;
use sc_executor_common::{wasm_runtime::WasmModule, runtime_blob::RuntimeBlob};
use tracing_subscriber::layer::SubscriberExt;
use crate::WasmExecutionMethod;
@@ -77,13 +79,12 @@ fn call_in_wasm<E: Externalities>(
8,
None,
);
executor.call_in_wasm(
&wasm_binary_unwrap()[..],
None,
executor.uncached_call(
RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
ext,
true,
function,
call_data,
ext,
sp_core::traits::MissingHostFunctions::Allow,
)
}
@@ -541,28 +542,37 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) {
None,
);
let err = executor.call_in_wasm(
&wasm_binary_unwrap()[..],
None,
"test_exhaust_heap",
&[0],
&mut ext.ext(),
sp_core::traits::MissingHostFunctions::Allow,
).unwrap_err();
let err = executor
.uncached_call(
RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
&mut ext.ext(),
true,
"test_exhaust_heap",
&[0],
)
.unwrap_err();
assert!(err.contains("Allocator ran out of space"));
}
test_wasm_execution!(returns_mutable_static);
fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
fn mk_test_runtime(wasm_method: WasmExecutionMethod, pages: u64) -> Arc<dyn WasmModule> {
let blob = RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..])
.expect("failed to create a runtime blob out of test runtime");
crate::wasm_runtime::create_wasm_runtime_with_code(
wasm_method,
1024,
&wasm_binary_unwrap()[..],
pages,
blob,
HostFunctions::host_functions(),
true,
None,
).expect("Creates runtime");
)
.expect("failed to instantiate wasm runtime")
}
test_wasm_execution!(returns_mutable_static);
fn returns_mutable_static(wasm_method: WasmExecutionMethod) {
let runtime = mk_test_runtime(wasm_method, 1024);
let instance = runtime.new_instance().unwrap();
let res = instance.call_export("returns_mutable_static", &[0]).unwrap();
@@ -589,14 +599,7 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
// to our allocator algorithm there are inefficiencies.
const REQUIRED_MEMORY_PAGES: u64 = 32;
let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
wasm_method,
REQUIRED_MEMORY_PAGES,
&wasm_binary_unwrap()[..],
HostFunctions::host_functions(),
true,
None,
).expect("Creates runtime");
let runtime = mk_test_runtime(wasm_method, REQUIRED_MEMORY_PAGES);
let instance = runtime.new_instance().unwrap();
// On the first invocation we allocate approx. 768KB (75%) of stack and then trap.
@@ -610,14 +613,7 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) {
test_wasm_execution!(interpreted_only heap_is_reset_between_calls);
fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) {
let runtime = crate::wasm_runtime::create_wasm_runtime_with_code(
wasm_method,
1024,
&wasm_binary_unwrap()[..],
HostFunctions::host_functions(),
true,
None,
).expect("Creates runtime");
let runtime = mk_test_runtime(wasm_method, 1024);
let instance = runtime.new_instance().unwrap();
let heap_base = instance.get_global_const("__heap_base")
@@ -642,27 +638,27 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) {
8,
None,
));
let code_hash = blake2_256(wasm_binary_unwrap()).to_vec();
let threads: Vec<_> = (0..8).map(|_|
{
let threads: Vec<_> = (0..8)
.map(|_| {
let executor = executor.clone();
let code_hash = code_hash.clone();
std::thread::spawn(move || {
let mut ext = TestExternalities::default();
let mut ext = ext.ext();
assert_eq!(
executor.call_in_wasm(
&wasm_binary_unwrap()[..],
Some(code_hash.clone()),
"test_twox_128",
&[0],
&mut ext,
sp_core::traits::MissingHostFunctions::Allow,
).unwrap(),
executor
.uncached_call(
RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
&mut ext,
true,
"test_twox_128",
&[0],
)
.unwrap(),
hex!("99e9d85137db46ef4bbea33613baafd5").to_vec().encode(),
);
})
}).collect();
})
.collect();
for t in threads.into_iter() {
t.join().unwrap();
@@ -671,9 +667,7 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) {
test_wasm_execution!(wasm_tracing_should_work);
fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) {
use std::sync::{Arc, Mutex};
use std::sync::Mutex;
use sc_tracing::{SpanDatum, TraceEvent};
struct TestTraceHandler(Arc<Mutex<Vec<SpanDatum>>>);
@@ -779,6 +773,5 @@ fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecu
&mut ext,
).unwrap_err();
dbg!(&error_result);
assert!(format!("{}", error_result).contains("Spawned task"));
}
+15 -11
View File
@@ -38,14 +38,17 @@ mod wasm_runtime;
mod integration_tests;
pub use wasmi;
pub use native_executor::{with_externalities_safe, NativeExecutor, WasmExecutor, NativeExecutionDispatch};
pub use native_executor::{
with_externalities_safe, NativeExecutor, WasmExecutor, NativeExecutionDispatch,
};
pub use sp_version::{RuntimeVersion, NativeVersion};
pub use codec::Codec;
#[doc(hidden)]
pub use sp_core::traits::{Externalities, CallInWasm};
pub use sp_core::traits::{Externalities};
#[doc(hidden)]
pub use sp_wasm_interface;
pub use wasm_runtime::WasmExecutionMethod;
pub use wasm_runtime::read_embedded_version;
pub use sc_executor_common::{error, sandbox};
@@ -68,7 +71,7 @@ mod tests {
use sc_runtime_test::wasm_binary_unwrap;
use sp_io::TestExternalities;
use sp_wasm_interface::HostFunctions;
use sp_core::traits::CallInWasm;
use sc_executor_common::runtime_blob::RuntimeBlob;
#[test]
fn call_in_interpreted_wasm_works() {
@@ -82,14 +85,15 @@ mod tests {
8,
None,
);
let res = executor.call_in_wasm(
&wasm_binary_unwrap()[..],
None,
"test_empty_return",
&[],
&mut ext,
sp_core::traits::MissingHostFunctions::Allow,
).unwrap();
let res = executor
.uncached_call(
RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(),
&mut ext,
true,
"test_empty_return",
&[],
)
.unwrap();
assert_eq!(res, vec![0u8; 0]);
}
}
@@ -33,14 +33,14 @@ use sp_version::{NativeVersion, RuntimeVersion};
use codec::{Decode, Encode};
use sp_core::{
NativeOrEncoded,
traits::{
CodeExecutor, Externalities, RuntimeCode, MissingHostFunctions,
RuntimeSpawnExt, RuntimeSpawn,
},
traits::{CodeExecutor, Externalities, RuntimeCode, RuntimeSpawnExt, RuntimeSpawn},
};
use log::trace;
use sp_wasm_interface::{HostFunctions, Function};
use sc_executor_common::wasm_runtime::{WasmInstance, WasmModule, InvokeMethod};
use sc_executor_common::{
wasm_runtime::{WasmInstance, WasmModule, InvokeMethod},
runtime_blob::RuntimeBlob,
};
use sp_externalities::ExternalitiesExt as _;
use sp_tasks::new_async_externalities;
@@ -188,64 +188,81 @@ impl WasmExecutor {
Err(e) => Err(e),
}
}
/// Perform a call into the given runtime.
///
/// The runtime is passed as a [`RuntimeBlob`]. The runtime will be isntantiated with the
/// parameters this `WasmExecutor` was initialized with.
///
/// In case of problems with during creation of the runtime or instantation, a `Err` is returned.
/// that describes the message.
#[doc(hidden)] // We use this function for tests across multiple crates.
pub fn uncached_call(
&self,
runtime_blob: RuntimeBlob,
ext: &mut dyn Externalities,
allow_missing_host_functions: bool,
export_name: &str,
call_data: &[u8],
) -> std::result::Result<Vec<u8>, String> {
let module = crate::wasm_runtime::create_wasm_runtime_with_code(
self.method,
self.default_heap_pages,
runtime_blob,
self.host_functions.to_vec(),
allow_missing_host_functions,
self.cache_path.as_deref(),
)
.map_err(|e| format!("Failed to create module: {:?}", e))?;
let instance = module
.new_instance()
.map_err(|e| format!("Failed to create instance: {:?}", e))?;
let instance = AssertUnwindSafe(instance);
let mut ext = AssertUnwindSafe(ext);
let module = AssertUnwindSafe(module);
with_externalities_safe(&mut **ext, move || {
preregister_builtin_ext(module.clone());
instance.call_export(export_name, call_data)
})
.and_then(|r| r)
.map_err(|e| e.to_string())
}
}
impl sp_core::traits::CallInWasm for WasmExecutor {
fn call_in_wasm(
impl sp_core::traits::ReadRuntimeVersion for WasmExecutor {
fn read_runtime_version(
&self,
wasm_code: &[u8],
code_hash: Option<Vec<u8>>,
method: &str,
call_data: &[u8],
ext: &mut dyn Externalities,
missing_host_functions: MissingHostFunctions,
) -> std::result::Result<Vec<u8>, String> {
let allow_missing_host_functions = missing_host_functions.allowed();
let runtime_blob = RuntimeBlob::uncompress_if_needed(&wasm_code)
.map_err(|e| format!("Failed to create runtime blob: {:?}", e))?;
if let Some(hash) = code_hash {
let code = RuntimeCode {
code_fetcher: &sp_core::traits::WrappedRuntimeCode(wasm_code.into()),
hash,
heap_pages: None,
};
self.with_instance(&code, ext, allow_missing_host_functions, |module, instance, _, mut ext| {
with_externalities_safe(
&mut **ext,
move || {
RuntimeInstanceSpawn::register_on_externalities(module.clone());
instance.call_export(method, call_data)
}
)
}).map_err(|e| e.to_string())
} else {
let module = crate::wasm_runtime::create_wasm_runtime_with_code(
self.method,
self.default_heap_pages,
&wasm_code,
self.host_functions.to_vec(),
allow_missing_host_functions,
self.cache_path.as_deref(),
)
.map_err(|e| format!("Failed to create module: {:?}", e))?;
let instance = module.new_instance()
.map_err(|e| format!("Failed to create instance: {:?}", e))?;
let instance = AssertUnwindSafe(instance);
let mut ext = AssertUnwindSafe(ext);
let module = AssertUnwindSafe(module);
with_externalities_safe(
&mut **ext,
move || {
RuntimeInstanceSpawn::register_on_externalities(module.clone());
instance.call_export(method, call_data)
}
)
.and_then(|r| r)
.map_err(|e| e.to_string())
if let Some(version) = crate::wasm_runtime::read_embedded_version(&runtime_blob)
.map_err(|e| format!("Failed to read the static section: {:?}", e))
.map(|v| v.map(|v| v.encode()))?
{
return Ok(version);
}
// If the blob didn't have embedded runtime version section, we fallback to the legacy
// way of fetching the verison: i.e. instantiating the given instance and calling
// `Core_version` on it.
self.uncached_call(
runtime_blob,
ext,
// If a runtime upgrade introduces new host functions that are not provided by
// the node, we should not fail at instantiation. Otherwise nodes that are
// updated could run this successfully and it could lead to a storage root
// mismatch when importing this block.
true,
"Core_version",
&[],
)
}
}
@@ -436,29 +453,25 @@ impl RuntimeInstanceSpawn {
ext.extension::<sp_core::traits::TaskExecutorExt>()
.map(move |task_ext| Self::new(module, task_ext.clone()))
}
}
/// Register new `RuntimeSpawnExt` on current externalities.
///
/// This extensions will spawn instances from provided `module`.
pub fn register_on_externalities(module: Arc<dyn WasmModule>) {
sp_externalities::with_externalities(
move |mut ext| {
if let Some(runtime_spawn) =
Self::with_externalities_and_module(module.clone(), ext)
{
if let Err(e) = ext.register_extension(
RuntimeSpawnExt(Box::new(runtime_spawn))
) {
trace!(
target: "executor",
"Failed to register `RuntimeSpawnExt` instance on externalities: {:?}",
e,
)
}
}
/// Pre-registers the built-in extensions to the currently effective externalities.
///
/// Meant to be called each time before calling into the runtime.
fn preregister_builtin_ext(module: Arc<dyn WasmModule>) {
sp_externalities::with_externalities(move |mut ext| {
if let Some(runtime_spawn) =
RuntimeInstanceSpawn::with_externalities_and_module(module, ext)
{
if let Err(e) = ext.register_extension(RuntimeSpawnExt(Box::new(runtime_spawn))) {
trace!(
target: "executor",
"Failed to register `RuntimeSpawnExt` instance on externalities: {:?}",
e,
)
}
);
}
}
});
}
impl<D: NativeExecutionDispatch + 'static> CodeExecutor for NativeExecutor<D> {
@@ -506,7 +519,7 @@ impl<D: NativeExecutionDispatch + 'static> CodeExecutor for NativeExecutor<D> {
with_externalities_safe(
&mut **ext,
move || {
RuntimeInstanceSpawn::register_on_externalities(module.clone());
preregister_builtin_ext(module.clone());
instance.call_export(method, data).map(NativeOrEncoded::Encoded)
}
)
@@ -557,17 +570,13 @@ impl<D: NativeExecutionDispatch> Clone for NativeExecutor<D> {
}
}
impl<D: NativeExecutionDispatch> sp_core::traits::CallInWasm for NativeExecutor<D> {
fn call_in_wasm(
impl<D: NativeExecutionDispatch> sp_core::traits::ReadRuntimeVersion for NativeExecutor<D> {
fn read_runtime_version(
&self,
wasm_blob: &[u8],
code_hash: Option<Vec<u8>>,
method: &str,
call_data: &[u8],
wasm_code: &[u8],
ext: &mut dyn Externalities,
missing_host_functions: MissingHostFunctions,
) -> std::result::Result<Vec<u8>, String> {
self.wasm.call_in_wasm(wasm_blob, code_hash, method, call_data, ext, missing_host_functions)
self.wasm.read_runtime_version(wasm_code, ext)
}
}
+102 -37
View File
@@ -29,7 +29,10 @@ use sp_core::traits::{Externalities, RuntimeCode, FetchRuntimeCode};
use sp_version::RuntimeVersion;
use std::panic::AssertUnwindSafe;
use std::path::{Path, PathBuf};
use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance};
use sc_executor_common::{
wasm_runtime::{WasmModule, WasmInstance},
runtime_blob::RuntimeBlob,
};
use sp_wasm_interface::Function;
@@ -235,6 +238,9 @@ impl RuntimeCache {
None => {
let code = runtime_code.fetch_runtime_code().ok_or(WasmError::CodeNotFound)?;
#[cfg(not(target_os = "unknown"))]
let time = std::time::Instant::now();
let result = create_versioned_wasm_runtime(
&code,
code_hash.clone(),
@@ -246,9 +252,22 @@ impl RuntimeCache {
self.max_runtime_instances,
self.cache_path.as_deref(),
);
if let Err(ref err) = result {
log::warn!(target: "wasm-runtime", "Cannot create a runtime: {:?}", err);
match result {
Ok(ref result) => {
#[cfg(not(target_os = "unknown"))]
log::debug!(
target: "wasm-runtime",
"Prepared new runtime version {:?} in {} ms.",
result.version,
time.elapsed().as_millis(),
);
}
Err(ref err) => {
log::warn!(target: "wasm-runtime", "Cannot create a runtime: {:?}", err);
}
}
Arc::new(result?)
}
};
@@ -278,16 +297,11 @@ impl RuntimeCache {
pub fn create_wasm_runtime_with_code(
wasm_method: WasmExecutionMethod,
heap_pages: u64,
code: &[u8],
blob: RuntimeBlob,
host_functions: Vec<&'static dyn Function>,
allow_missing_func_imports: bool,
cache_path: Option<&Path>,
) -> Result<Arc<dyn WasmModule>, WasmError> {
use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
let code = sp_maybe_compressed_blob::decompress(code, CODE_BLOB_BOMB_LIMIT)
.map_err(|e| WasmError::Other(format!("Decompression error: {:?}", e)))?;
match wasm_method {
WasmExecutionMethod::Interpreted => {
// Wasmi doesn't have any need in a cache directory.
@@ -297,7 +311,7 @@ pub fn create_wasm_runtime_with_code(
drop(cache_path);
sc_executor_wasmi::create_runtime(
&code,
blob,
heap_pages,
host_functions,
allow_missing_func_imports,
@@ -306,7 +320,6 @@ pub fn create_wasm_runtime_with_code(
}
#[cfg(feature = "wasmtime")]
WasmExecutionMethod::Compiled => {
let blob = sc_executor_common::runtime_blob::RuntimeBlob::new(&code)?;
sc_executor_wasmtime::create_runtime(
sc_executor_wasmtime::CodeSupplyMode::Verbatim { blob },
sc_executor_wasmtime::Config {
@@ -343,6 +356,55 @@ fn decode_version(version: &[u8]) -> Result<RuntimeVersion, WasmError> {
}
}
fn decode_runtime_apis(apis: &[u8]) -> Result<Vec<([u8; 8], u32)>, WasmError> {
use std::convert::TryFrom;
use sp_api::RUNTIME_API_INFO_SIZE;
apis.chunks(RUNTIME_API_INFO_SIZE)
.map(|chunk| {
// `chunk` can be less than `RUNTIME_API_INFO_SIZE` if the total length of `apis` doesn't
// completely divide by `RUNTIME_API_INFO_SIZE`.
<[u8; RUNTIME_API_INFO_SIZE]>::try_from(chunk)
.map(sp_api::deserialize_runtime_api_info)
.map_err(|_| {
WasmError::Other(format!(
"a clipped runtime api info declaration"
))
})
})
.collect::<Result<Vec<_>, WasmError>>()
}
/// Take the runtime blob and scan it for the custom wasm sections containing the version information
/// and construct the `RuntimeVersion` from them.
///
/// If there are no such sections, it returns `None`. If there is an error during decoding those
/// sections, `Err` will be returned.
pub fn read_embedded_version(
blob: &RuntimeBlob,
) -> Result<Option<RuntimeVersion>, WasmError> {
if let Some(version_section) = blob.custom_section_contents("runtime_version") {
// We do not use `decode_version` here because the runtime_version section is not supposed
// to ever contain a legacy version. Apart from that `decode_version` relies on presence
// of a special API in the `apis` field to treat the input as a non-legacy version. However
// the structure found in the `runtime_version` always contain an empty `apis` field. Therefore
// the version read will be mistakingly treated as an legacy one.
let mut decoded_version = sp_api::RuntimeVersion::decode(&mut &version_section[..])
.map_err(|_|
WasmError::Instantiation("failed to decode verison section".into())
)?;
// Don't stop on this and check if there is a special section that encodes all runtime APIs.
if let Some(apis_section) = blob.custom_section_contents("runtime_apis") {
decoded_version.apis = decode_runtime_apis(apis_section)?.into();
}
Ok(Some(decoded_version))
} else {
Ok(None)
}
}
fn create_versioned_wasm_runtime(
code: &[u8],
code_hash: Vec<u8>,
@@ -354,41 +416,44 @@ fn create_versioned_wasm_runtime(
max_instances: usize,
cache_path: Option<&Path>,
) -> Result<VersionedRuntime, WasmError> {
#[cfg(not(target_os = "unknown"))]
let time = std::time::Instant::now();
// The incoming code may be actually compressed. We decompress it here and then work with
// the uncompressed code from now on.
let blob = sc_executor_common::runtime_blob::RuntimeBlob::uncompress_if_needed(&code)?;
// Use the runtime blob to scan if there is any metadata embedded into the wasm binary pertaining
// to runtime version. We do it before consuming the runtime blob for creating the runtime.
let mut version: Option<_> = read_embedded_version(&blob)?;
let runtime = create_wasm_runtime_with_code(
wasm_method,
heap_pages,
&code,
blob,
host_functions,
allow_missing_func_imports,
cache_path,
)?;
// Call to determine runtime version.
let version_result = {
// `ext` is already implicitly handled as unwind safe, as we store it in a global variable.
let mut ext = AssertUnwindSafe(ext);
// If the runtime blob doesn't embed the runtime version then use the legacy version query
// mechanism: call the runtime.
if version.is_none() {
// Call to determine runtime version.
let version_result = {
// `ext` is already implicitly handled as unwind safe, as we store it in a global variable.
let mut ext = AssertUnwindSafe(ext);
// The following unwind safety assertion is OK because if the method call panics, the
// runtime will be dropped.
let runtime = AssertUnwindSafe(runtime.as_ref());
crate::native_executor::with_externalities_safe(
&mut **ext,
move || runtime.new_instance()?.call("Core_version".into(), &[])
).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))?
};
let version = match version_result {
Ok(version) => Some(decode_version(&version)?),
Err(_) => None,
};
#[cfg(not(target_os = "unknown"))]
log::debug!(
target: "wasm-runtime",
"Prepared new runtime version {:?} in {} ms.",
version,
time.elapsed().as_millis(),
);
// The following unwind safety assertion is OK because if the method call panics, the
// runtime will be dropped.
let runtime = AssertUnwindSafe(runtime.as_ref());
crate::native_executor::with_externalities_safe(
&mut **ext,
move || runtime.new_instance()?.call("Core_version".into(), &[])
).map_err(|_| WasmError::Instantiation("panic in call to get runtime version".into()))?
};
if let Ok(version_buf) = version_result {
version = Some(decode_version(&version_buf)?)
}
}
let mut instances = Vec::with_capacity(max_instances);
instances.resize_with(max_instances, || Mutex::new(None));