Re-introduce zero copy codec and add minimal polkadot client API which uses linked native runtime (#65)

* client-api type and move duty roster types to primitives

* tuple implementation for slicable

* mild cleanup of deserialization code

* stubs which handle encoding and decoding themselves

* fancier impl_stubs macro

* zero-copy slicable API

* minimal polkadot-client API

* fix WASM API generation

* move native environment stuff to substrate executor

* fix warnings and grumbles
This commit is contained in:
Robert Habermeier
2018-02-08 19:20:34 +01:00
committed by Gav Wood
parent f2b3bab61e
commit a00d0e75fd
31 changed files with 634 additions and 252 deletions
+19 -13
View File
@@ -1,20 +1,20 @@
// Copyright 2017 Parity Technologies (UK) Ltd.
// This file is part of Polkadot.
// This file is part of Substrate.
// Polkadot is free software: you can redistribute it and/or modify
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Polkadot is distributed in the hope that it will be useful,
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Polkadot Client
//! Substrate Client
#![warn(missing_docs)]
@@ -139,8 +139,9 @@ impl<B, E> Client<B, E> where
})
}
fn state_at(&self, id: BlockId) -> error::Result<B::State> {
self.backend.state_at(id)
/// Get a reference to the state at a given block.
pub fn state_at(&self, block: &BlockId) -> error::Result<B::State> {
self.backend.state_at(*block)
}
/// Expose backend reference. To be used in tests only
@@ -148,28 +149,33 @@ impl<B, E> Client<B, E> where
&self.backend
}
/// Return single storage entry of contract under given address in state in a block of given id.
/// Return single storage entry of contract under given address in state in a block of given hash.
pub fn storage(&self, id: &BlockId, key: &StorageKey) -> error::Result<StorageData> {
Ok(self.state_at(*id)?
Ok(self.state_at(id)?
.storage(&key.0)
.map(|x| StorageData(x.to_vec()))?)
}
/// Execute a call to a contract on top of state in a block of given id.
/// Get the code at a given block.
pub fn code_at(&self, id: &BlockId) -> error::Result<Vec<u8>> {
self.storage(id, &StorageKey(b":code:".to_vec())).map(|data| data.0)
}
/// Execute a call to a contract on top of state in a block of given hash.
///
/// No changes are made.
pub fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<CallResult> {
let state = self.state_at(*id)?;
let mut changes = state_machine::OverlayedChanges::default();
let state = self.state_at(id)?;
let _ = state_machine::execute(
let return_data = state_machine::execute(
&state,
&mut changes,
&self.executor,
method,
call_data,
)?;
Ok(CallResult { return_data: vec![], changes })
Ok(CallResult { return_data, changes })
}
/// Queue a block for import.
+2 -1
View File
@@ -20,6 +20,7 @@
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(not(feature = "std"), feature(alloc))]
#[cfg_attr(not(feature = "std"), macro_use)]
extern crate substrate_runtime_std as rstd;
mod endiansensitive;
@@ -28,6 +29,6 @@ mod joiner;
mod keyedvec;
pub use self::endiansensitive::EndianSensitive;
pub use self::slicable::{Slicable, NonTrivialSlicable};
pub use self::slicable::{Input, Slicable, NonTrivialSlicable};
pub use self::joiner::Joiner;
pub use self::keyedvec::KeyedVec;
+160 -24
View File
@@ -21,13 +21,25 @@ use rstd::vec::Vec;
use super::joiner::Joiner;
use super::endiansensitive::EndianSensitive;
/// Trait that allows reading of data into a slice.
pub trait Input {
/// Read into the provided input slice. Returns the number of bytes read.
fn read(&mut self, into: &mut [u8]) -> usize;
}
impl<'a> Input for &'a [u8] {
fn read(&mut self, into: &mut [u8]) -> usize {
let len = ::rstd::cmp::min(into.len(), self.len());
into[..len].copy_from_slice(&self[..len]);
*self = &self[len..];
len
}
}
/// Trait that allows zero-copy read/write of value-references to/from slices in LE format.
pub trait Slicable: Sized {
/// Attempt to deserialise the value from a slice. Ignore trailing bytes and
/// set the slice's start to just after the last byte consumed.
///
/// If `None` is returned, then the slice should be unmodified.
fn from_slice(value: &mut &[u8]) -> Option<Self>;
/// Attempt to deserialise the value from input.
fn decode<I: Input>(value: &mut I) -> Option<Self>;
/// Convert self to an owned vector.
fn to_vec(&self) -> Vec<u8> {
self.as_slice_then(|s| s.to_vec())
@@ -40,16 +52,19 @@ pub trait Slicable: Sized {
pub trait NonTrivialSlicable: Slicable {}
impl<T: EndianSensitive> Slicable for T {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
let size = mem::size_of::<T>();
assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type.");
if value.len() >= size {
let x: T = unsafe { ::rstd::ptr::read(value.as_ptr() as *const T) };
*value = &value[size..];
Some(x.from_le())
} else {
None
let mut val: T = unsafe { mem::zeroed() };
unsafe {
let raw: &mut [u8] = slice::from_raw_parts_mut(
&mut val as *mut T as *mut u8,
size
);
if input.read(raw) != size { return None }
}
Some(val.from_le())
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -70,12 +85,15 @@ impl<T: EndianSensitive> Slicable for T {
}
impl Slicable for Vec<u8> {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
u32::from_slice(value).map(move |len| {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
u32::decode(input).and_then(move |len| {
let len = len as usize;
let res = value[..len].to_vec();
*value = &value[len..];
res
let mut vec = vec![0; len];
if input.read(&mut vec[..len]) != len {
None
} else {
Some(vec)
}
})
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -92,18 +110,60 @@ impl Slicable for Vec<u8> {
}
}
macro_rules! impl_vec_simple_array {
($($size:expr),*) => {
$(
impl<T> Slicable for Vec<[T; $size]>
where [T; $size]: EndianSensitive
{
fn decode<I: Input>(input: &mut I) -> Option<Self> {
u32::decode(input).and_then(move |len| {
let mut r = Vec::with_capacity(len as usize);
for _ in 0..len {
r.push(match Slicable::decode(input) {
Some(x) => x,
None => return None,
});
}
Some(r)
})
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
f(&self.to_vec())
}
fn to_vec(&self) -> Vec<u8> {
use rstd::iter::Extend;
let len = self.len();
assert!(len <= u32::max_value() as usize, "Attempted to serialize vec with too many elements.");
let mut r: Vec<u8> = Vec::new().join(&(len as u32));
for item in self {
r.extend(item.to_vec());
}
r
}
}
)*
}
}
impl_vec_simple_array!(1, 2, 4, 8, 16, 32, 64);
impl<T: Slicable> NonTrivialSlicable for Vec<T> where Vec<T>: Slicable {}
impl<T: NonTrivialSlicable> Slicable for Vec<T> {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
u32::from_slice(value).and_then(move |len| {
let len = len as usize;
let mut r = Vec::with_capacity(len);
fn decode<I: Input>(input: &mut I) -> Option<Self> {
u32::decode(input).and_then(move |len| {
let mut r = Vec::with_capacity(len as usize);
for _ in 0..len {
match T::from_slice(value) {
r.push(match T::decode(input) {
Some(x) => x,
None => return None,
Some(v) => r.push(v),
}
});
}
Some(r)
@@ -128,6 +188,82 @@ impl<T: NonTrivialSlicable> Slicable for Vec<T> {
}
}
impl Slicable for () {
fn decode<I: Input>(_: &mut I) -> Option<()> {
Some(())
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
f(&[])
}
fn to_vec(&self) -> Vec<u8> {
Vec::new()
}
}
macro_rules! tuple_impl {
($one:ident,) => {
impl<$one: Slicable> Slicable for ($one,) {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
match $one::decode(input) {
None => None,
Some($one) => Some(($one,)),
}
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
self.0.as_slice_then(f)
}
}
};
($first:ident, $($rest:ident,)+) => {
impl<$first: Slicable, $($rest: Slicable),+>
Slicable for
($first, $($rest),+) {
fn decode<INPUT: Input>(input: &mut INPUT) -> Option<Self> {
Some((
match $first::decode(input) {
Some(x) => x,
None => return None,
},
$(match $rest::decode(input) {
Some(x) => x,
None => return None,
},)+
))
}
fn as_slice_then<RETURN, PROCESS>(&self, f: PROCESS) -> RETURN
where PROCESS: FnOnce(&[u8]) -> RETURN
{
let mut v = Vec::new();
let (
ref $first,
$(ref $rest),+
) = *self;
$first.as_slice_then(|s| v.extend(s));
$($rest.as_slice_then(|s| v.extend(s));)+
f(v.as_slice())
}
}
tuple_impl!($($rest,)+);
}
}
#[allow(non_snake_case)]
mod inner_tuple_impl {
use rstd::vec::Vec;
use super::{Input, Slicable};
tuple_impl!(A, B, C, D, E, F, G, H, I, J, K,);
}
#[cfg(test)]
mod tests {
use super::*;
+1 -1
View File
@@ -61,4 +61,4 @@ mod native_executor;
pub mod error;
pub use wasm_executor::WasmExecutor;
pub use native_executor::{NativeExecutionDispatch, NativeExecutor};
pub use native_executor::{with_native_environment, NativeExecutor, NativeExecutionDispatch};
@@ -15,11 +15,23 @@
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
use error::{Error, ErrorKind, Result};
use runtime_io;
use state_machine::{Externalities, CodeExecutor};
use wasm_executor::WasmExecutor;
use std::panic::catch_unwind;
fn safe_call<F, U>(f: F) -> Result<U>
where F: ::std::panic::UnwindSafe + FnOnce() -> U
{
::std::panic::catch_unwind(f).map_err(|_| ErrorKind::Runtime.into())
}
/// Set up the externalities and safe calling environment to execute calls to a native runtime.
///
/// If the inner closure panics, it will be caught and return an error.
pub fn with_native_environment<F, U>(ext: &mut Externalities, f: F) -> Result<U>
where F: ::std::panic::UnwindSafe + FnOnce() -> U
{
::runtime_io::with_externalities(ext, move || safe_call(f))
}
/// Delegate for dispatching a CodeExecutor call to native code.
pub trait NativeExecutionDispatch {
@@ -28,7 +40,7 @@ pub trait NativeExecutionDispatch {
/// Dispatch a method and input data to be executed natively. Returns `Some` result or `None`
/// if the `method` is unknown. Panics if there's an unrecoverable error.
fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>>;
fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>>;
}
/// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence
@@ -38,10 +50,6 @@ pub struct NativeExecutor<D: NativeExecutionDispatch + Sync + Send> {
pub _dummy: ::std::marker::PhantomData<D>,
}
fn safe_call<F: ::std::panic::UnwindSafe + FnOnce() -> Option<Vec<u8>>>(f: F) -> Result<Option<Vec<u8>>> {
catch_unwind(f).map_err(|_| ErrorKind::Runtime.into())
}
impl<D: NativeExecutionDispatch + Sync + Send> CodeExecutor for NativeExecutor<D> {
type Error = Error;
@@ -53,8 +61,8 @@ impl<D: NativeExecutionDispatch + Sync + Send> CodeExecutor for NativeExecutor<D
data: &[u8],
) -> Result<Vec<u8>> {
if code == D::native_equivalent() {
runtime_io::with_externalities(ext, || safe_call(|| D::dispatch(method, data)))?
.ok_or(ErrorKind::MethodNotFound(method.to_owned()).into())
// call native
D::dispatch(ext, method, data)
} else {
// call into wasm.
WasmExecutor.call(ext, code, method, data)
+17 -17
View File
@@ -19,7 +19,7 @@
#[cfg(feature = "std")]
use bytes;
use rstd::vec::Vec;
use codec::Slicable;
use codec::{Input, Slicable};
use hash::H256;
/// Used to refer to a block number.
@@ -37,8 +37,8 @@ pub type TransactionHash = H256;
pub struct Transaction(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
impl Slicable for Transaction {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
Vec::<u8>::from_slice(value).map(Transaction)
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Vec::<u8>::decode(input).map(Transaction)
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -54,8 +54,8 @@ impl ::codec::NonTrivialSlicable for Transaction { }
pub struct Log(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
impl Slicable for Log {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
Vec::<u8>::from_slice(value).map(Log)
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Vec::<u8>::decode(input).map(Log)
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -74,8 +74,8 @@ pub struct Digest {
}
impl Slicable for Digest {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
Vec::<Log>::from_slice(value).map(|logs| Digest { logs })
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Vec::<Log>::decode(input).map(|logs| Digest { logs })
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -97,10 +97,10 @@ pub struct Block {
}
impl Slicable for Block {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Some(Block {
header: try_opt!(Slicable::from_slice(value)),
transactions: try_opt!(Slicable::from_slice(value)),
header: try_opt!(Slicable::decode(input)),
transactions: try_opt!(Slicable::decode(input)),
})
}
@@ -152,13 +152,13 @@ impl Header {
}
impl Slicable for Header {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
Some(Header {
parent_hash: try_opt!(Slicable::from_slice(value)),
number: try_opt!(Slicable::from_slice(value)),
state_root: try_opt!(Slicable::from_slice(value)),
transaction_root: try_opt!(Slicable::from_slice(value)),
digest: try_opt!(Slicable::from_slice(value)),
parent_hash: try_opt!(Slicable::decode(input)),
number: try_opt!(Slicable::decode(input)),
state_root: try_opt!(Slicable::decode(input)),
transaction_root: try_opt!(Slicable::decode(input)),
digest: try_opt!(Slicable::decode(input)),
})
}
@@ -208,6 +208,6 @@ mod tests {
}"#);
let v = header.to_vec();
assert_eq!(Header::from_slice(&mut &v[..]).unwrap(), header);
assert_eq!(Header::decode(&mut &v[..]).unwrap(), header);
}
}
+2 -2
View File
@@ -40,8 +40,8 @@ macro_rules! impl_rest {
}
impl ::codec::Slicable for $name {
fn from_slice(value: &mut &[u8]) -> Option<Self> {
<[u8; $len] as ::codec::Slicable>::from_slice(value).map($name)
fn decode<I: ::codec::Input>(input: &mut I) -> Option<Self> {
<[u8; $len] as ::codec::Slicable>::decode(input).map($name)
}
fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+15 -4
View File
@@ -22,6 +22,8 @@ extern crate substrate_primitives as primitives;
extern crate triehash;
extern crate ed25519;
#[doc(hidden)]
pub extern crate substrate_codec as codec;
// re-export hashing functions.
pub use primitives::{blake2_256, twox_128, twox_256};
@@ -40,7 +42,7 @@ pub fn storage(key: &[u8]) -> Vec<u8> {
}
/// Get `key` from storage, placing the value into `value_out` (as much as possible) and return
/// the number of bytes that the key in storage was.
/// the number of bytes that the key in storage was beyond the offset.
pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> usize {
ext::with(|ext| {
if let Ok(value) = ext.storage(key) {
@@ -124,11 +126,20 @@ pub fn print<T: Printable + Sized>(value: T) {
#[macro_export]
macro_rules! impl_stubs {
($( $name:ident ),*) => {
pub fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
( $( $name:ident => $invoke:expr ),* ) => {
/// Dispatch logic for the native runtime.
pub fn dispatch(method: &str, mut data: &[u8]) -> Option<Vec<u8>> {
match method {
$(
stringify!($name) => Some($name(data)),
stringify!($name) => {
let input = match $crate::codec::Slicable::decode(&mut data) {
Some(input) => input,
None => panic!("Bad input data provided to {}", stringify!($name)),
};
let output = $invoke(input);
Some($crate::codec::Slicable::to_vec(&output))
}
)*
_ => None,
}
+31 -20
View File
@@ -17,6 +17,9 @@
extern crate substrate_runtime_std as rstd;
extern crate substrate_primitives as primitives;
#[doc(hidden)]
pub extern crate substrate_codec as codec;
use rstd::intrinsics;
use rstd::vec::Vec;
pub use rstd::{mem, slice};
@@ -68,7 +71,7 @@ pub fn set_storage(key: &[u8], value: &[u8]) {
}
/// Get `key` from storage, placing the value into `value_out` (as much as possible) and return
/// the number of bytes that the key in storage was.
/// the number of bytes that the key in storage was beyond the offset.
pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> usize {
unsafe {
ext_get_storage_into(
@@ -178,25 +181,33 @@ pub fn print<T: Printable + Sized>(value: T) {
#[macro_export]
macro_rules! impl_stubs {
( $( $name:ident ),* ) => {
pub mod _internal {
$(
#[no_mangle]
pub fn $name(input_data: *mut u8, input_len: usize) -> u64 {
let input = if input_len == 0 {
&[0u8; 0]
} else {
unsafe {
$crate::slice::from_raw_parts(input_data, input_len)
}
};
( $( $new_name:ident => $invoke:expr ),* ) => {
$(
#[no_mangle]
pub fn $new_name(input_data: *mut u8, input_len: usize) -> u64 {
let mut input = if input_len == 0 {
&[0u8; 0]
} else {
unsafe {
$crate::slice::from_raw_parts(input_data, input_len)
}
};
let output = super::$name(input);
let r = output.as_ptr() as u64 + ((output.len() as u64) << 32);
$crate::mem::forget(output);
r
}
)*
}
let input = match $crate::codec::Slicable::decode(&mut input) {
Some(input) => input,
None => panic!("Bad input data provided to {}", stringify!($name)),
};
let output = ($invoke)(input);
let output = $crate::codec::Slicable::to_vec(&output);
let res = output.as_ptr() as u64 + ((output.len() as u64) << 32);
// Leak the output vector to avoid it being freed.
// This is fine in a WASM context since the heap
// will be discarded after the call.
::core::mem::forget(output);
res
}
)*
}
}
@@ -21,6 +21,7 @@
#![cfg_attr(not(feature = "std"), feature(lang_items))]
#![cfg_attr(not(feature = "std"), feature(core_intrinsics))]
#![cfg_attr(not(feature = "std"), feature(alloc))]
#![cfg_attr(not(feature = "std"), feature(macro_reexport))]
#![cfg_attr(feature = "std", doc = "Polkadot runtime standard library as compiled when linked with Rust's standard library.")]
#![cfg_attr(not(feature = "std"), doc = "Polkadot's runtime standard library as compiled without Rust's standard library.")]
+6 -5
View File
@@ -14,12 +14,13 @@
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
pub use std::vec;
pub use std::rc;
pub use std::cell;
pub use std::boxed;
pub use std::slice;
pub use std::cell;
pub use std::cmp;
pub use std::iter;
pub use std::mem;
pub use std::ops;
pub use std::iter;
pub use std::ptr;
pub use std::rc;
pub use std::slice;
pub use std::vec;
@@ -15,19 +15,21 @@
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
#[cfg(feature = "nightly")]
#[macro_reexport(vec)]
extern crate alloc;
#[cfg(feature = "nightly")]
extern crate pwasm_libc;
#[cfg(feature = "nightly")]
extern crate pwasm_alloc;
pub use alloc::vec;
pub use alloc::boxed;
pub use alloc::rc;
pub use core::mem;
pub use core::slice;
pub use alloc::vec;
pub use core::cell;
pub use core::ops;
pub use core::iter;
pub use core::ptr;
pub use core::cmp;
pub use core::intrinsics;
pub use core::iter;
pub use core::mem;
pub use core::ops;
pub use core::ptr;
pub use core::slice;