Re-introduce zero copy codec and add minimal polkadot client API which uses linked native runtime (#65)

* client-api type and move duty roster types to primitives

* tuple implementation for slicable

* mild cleanup of deserialization code

* stubs which handle encoding and decoding themselves

* fancier impl_stubs macro

* zero-copy slicable API

* minimal polkadot-client API

* fix WASM API generation

* move native environment stuff to substrate executor

* fix warnings and grumbles

substrate/Cargo.lock

@@ -957,6 +957,19 @@ dependencies = [
  "polkadot-cli 0.1.0",
 ]
 
+[[package]]
+name = "polkadot-api"
+version = "0.1.0"
+dependencies = [
+ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "polkadot-executor 0.1.0",
+ "polkadot-primitives 0.1.0",
+ "polkadot-runtime 0.1.0",
+ "substrate-client 0.1.0",
+ "substrate-executor 0.1.0",
+ "substrate-state-machine 0.1.0",
+]
+
 [[package]]
 name = "polkadot-candidate-agreement"
 version = "0.1.0"

substrate/Cargo.toml

@@ -13,21 +13,22 @@ polkadot-cli = { path = "polkadot/cli" }
 
 [workspace]
 members = [
-	"substrate/client",
+	"polkadot/api",
-	"substrate/codec",
-	"substrate/environmental",
-	"substrate/executor",
 	"polkadot/candidate-agreement",
 	"polkadot/cli",
 	"polkadot/collator",
 	"polkadot/executor",
-	"polkadot/runtime",
 	"polkadot/primitives",
+	"polkadot/runtime",
 	"polkadot/validator",
+	"substrate/client",
+	"substrate/codec",
+	"substrate/environmental",
+	"substrate/executor",
 	"substrate/network",
 	"substrate/primitives",
-	"substrate/rpc",
 	"substrate/rpc-servers",
+	"substrate/rpc",
 	"substrate/runtime-io",
 	"substrate/runtime-std",
 	"substrate/serializer",

substrate/polkadot/api/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "polkadot-api"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+error-chain = "0.11"
+polkadot-executor = { path = "../executor" }
+polkadot-runtime = { path = "../runtime" }
+polkadot-primitives = { path = "../primitives" }
+substrate-client = { path = "../../substrate/client" }
+substrate-executor = { path = "../../substrate/executor" }
+substrate-state-machine = { path = "../../substrate/state-machine" }

substrate/polkadot/api/src/lib.rs

@@ -0,0 +1,118 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Strongly typed API for Polkadot based around the locally-compiled native
+//! runtime.
+
+extern crate polkadot_executor as polkadot_executor;
+extern crate polkadot_runtime ;
+extern crate polkadot_primitives as primitives;
+extern crate substrate_client as client;
+extern crate substrate_executor as substrate_executor;
+extern crate substrate_state_machine as state_machine;
+
+#[macro_use]
+extern crate error_chain;
+
+use client::backend::Backend;
+use client::blockchain::BlockId;
+use client::Client;
+use polkadot_runtime::runtime;
+use polkadot_executor::LocalNativeExecutionDispatch as LocalDispatch;
+use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
+use primitives::{AccountId, SessionKey};
+use primitives::parachain::DutyRoster;
+
+error_chain! {
+	errors {
+		/// Unknown runtime code.
+		UnknownRuntime {
+			description("Unknown runtime code")
+			display("Unknown runtime code")
+		}
+		UnknownBlock(b: BlockId) {
+			description("Unknown block")
+			display("Unknown block")
+		}
+		/// Some other error.
+		// TODO: allow to be specified as associated type of PolkadotApi
+		Other(e: Box<::std::error::Error + Send>) {
+			description("Other error")
+			display("Other error: {}", e.description())
+		}
+	}
+
+	links {
+		Executor(substrate_executor::error::Error, substrate_executor::error::ErrorKind);
+	}
+}
+
+/// Trait encapsulating the Polkadot API.
+///
+/// All calls should fail when the exact runtime is unknown.
+pub trait PolkadotApi {
+	/// Get authorities at a given block.
+	fn authorities(&self, at: &BlockId) -> Result<Vec<SessionKey>>;
+
+	/// Get validators at a given block.
+	fn validators(&self, at: &BlockId) -> Result<Vec<AccountId>>;
+
+	/// Get the authority duty roster at a block.
+	fn duty_roster(&self, at: &BlockId) -> Result<DutyRoster>;
+}
+
+fn convert_client_error(e: client::error::Error) -> Error {
+	match e {
+		client::error::Error(client::error::ErrorKind::UnknownBlock(b), _) => Error::from_kind(ErrorKind::UnknownBlock(b)),
+		other => Error::from_kind(ErrorKind::Other(Box::new(other) as Box<_>)),
+	}
+}
+
+// set up the necessary scaffolding to execute the runtime.
+macro_rules! with_runtime {
+	($client: ident, $at: expr, $exec: expr) => {{
+		// bail if the code is not the same as the natively linked.
+		if $client.code_at($at).map_err(convert_client_error)? != LocalDispatch::native_equivalent() {
+			bail!(ErrorKind::UnknownRuntime);
+		}
+
+		$client.state_at($at).map_err(convert_client_error).and_then(|state| {
+			let mut changes = Default::default();
+			let mut ext = state_machine::Ext {
+				overlay: &mut changes,
+				backend: &state,
+			};
+
+			::substrate_executor::with_native_environment(&mut ext, $exec).map_err(Into::into)
+		})
+	}}
+}
+
+impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
+	where ::client::error::Error: From<<<B as Backend>::State as state_machine::backend::Backend>::Error>
+{
+	fn authorities(&self, at: &BlockId) -> Result<Vec<SessionKey>> {
+		with_runtime!(self, at, ::runtime::consensus::authorities)
+	}
+
+	fn validators(&self, at: &BlockId) -> Result<Vec<AccountId>> {
+		with_runtime!(self, at, ::runtime::session::validators)
+	}
+
+	fn duty_roster(&self, at: &BlockId) -> Result<DutyRoster> {
+		with_runtime!(self, at, ::runtime::parachains::calculate_duty_roster)
+	}
+}

substrate/polkadot/cli/src/genesis.rs

@@ -89,7 +89,7 @@ mod tests {
 				"execute_transaction",
 				&vec![].join(&header).join(tx)
 			).unwrap();
-			header = Header::from_slice(&mut &ret_data[..]).unwrap();
+			header = Header::decode(&mut &ret_data[..]).unwrap();
 		}
 
 		let ret_data = execute(
@@ -99,7 +99,7 @@ mod tests {
 			"finalise_block",
 			&vec![].join(&header)
 		).unwrap();
-		header = Header::from_slice(&mut &ret_data[..]).unwrap();
+		header = Header::decode(&mut &ret_data[..]).unwrap();
 
 		(vec![].join(&Block { header, transactions }), hash.into())
 	}

substrate/polkadot/cli/src/lib.rs

@@ -83,7 +83,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 		storage = genesis_config.genesis_map();
 		let block = genesis::construct_genesis_block(&storage);
 		storage.extend(additional_storage_with_genesis(&block));
-		(primitives::block::Header::from_slice(&mut block.header.to_vec().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+		(primitives::block::Header::decode(&mut block.header.to_vec().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
 	};
 	let client = client::new_in_mem(executor, prepare_genesis)?;
 

substrate/polkadot/executor/src/lib.rs

@@ -31,7 +31,9 @@ extern crate triehash;
 extern crate hex_literal;
 
 use polkadot_runtime as runtime;
+use substrate_executor::error::{Error, ErrorKind};
 use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
+use state_machine::Externalities;
 
 /// A null struct which implements `NativeExecutionDispatch` feeding in the hard-coded runtime.
 pub struct LocalNativeExecutionDispatch;
@@ -43,8 +45,9 @@ impl NativeExecutionDispatch for LocalNativeExecutionDispatch {
 		include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm")
 	}
 
-	fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
+	fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
-		runtime::dispatch(method, data)
+		::substrate_executor::with_native_environment(ext, move || runtime::api::dispatch(method, data))?
+			.ok_or_else(|| ErrorKind::MethodNotFound(method.to_owned()).into())
 	}
 }
 
@@ -236,14 +239,6 @@ mod tests {
 		)
 	}
 
-	#[test]
-	fn test_execution_works() {
-		let mut t = new_test_ext();
-		println!("Testing Wasm...");
-		let r = WasmExecutor.call(&mut t, COMPACT_CODE, "run_tests", &block2().0);
-		assert!(r.is_ok());
-	}
-
 	#[test]
 	fn full_native_block_import_works() {
 		let mut t = new_test_ext();

substrate/polkadot/primitives/src/block.rs

@@ -20,7 +20,7 @@
 use primitives::bytes;
 use primitives::H256;
 use rstd::vec::Vec;
-use codec::Slicable;
+use codec::{Input, Slicable};
 use transaction::UncheckedTransaction;
 
 /// Used to refer to a block number.
@@ -38,8 +38,8 @@ pub type TransactionHash = H256;
 pub struct Log(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
 impl Slicable for Log {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Log)
+		Vec::<u8>::decode(input).map(Log)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -58,8 +58,8 @@ pub struct Digest {
 }
 
 impl Slicable for Digest {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<Log>::from_slice(value).map(|logs| Digest { logs })
+		Vec::<Log>::decode(input).map(|logs| Digest { logs })
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -81,11 +81,9 @@ pub struct Block {
 }
 
 impl Slicable for Block {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Some(Block {
+		let (header, transactions) = try_opt!(Slicable::decode(input));
-			header: try_opt!(Slicable::from_slice(value)),
+		Some(Block { header, transactions })
-			transactions: try_opt!(Slicable::from_slice(value)),
-		})
 	}
 
 	fn to_vec(&self) -> Vec<u8> {
@@ -136,13 +134,13 @@ impl Header {
 }
 
 impl Slicable for Header {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Header {
-			parent_hash: try_opt!(Slicable::from_slice(value)),
+			parent_hash: try_opt!(Slicable::decode(input)),
-			number: try_opt!(Slicable::from_slice(value)),
+			number: try_opt!(Slicable::decode(input)),
-			state_root: try_opt!(Slicable::from_slice(value)),
+			state_root: try_opt!(Slicable::decode(input)),
-			transaction_root: try_opt!(Slicable::from_slice(value)),
+			transaction_root: try_opt!(Slicable::decode(input)),
-			digest: try_opt!(Slicable::from_slice(value)),
+			digest: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -192,6 +190,6 @@ mod tests {
 }"#);
 
 		let v = header.to_vec();
-		assert_eq!(Header::from_slice(&mut &v[..]).unwrap(), header);
+		assert_eq!(Header::decode(&mut &v[..]).unwrap(), header);
 	}
 }

substrate/polkadot/primitives/src/lib.rs

@@ -29,11 +29,12 @@ extern crate serde_derive;
 extern crate serde;
 
 extern crate substrate_runtime_std as rstd;
-extern crate substrate_codec as codec;
 extern crate substrate_primitives as primitives;
 #[cfg(test)]
 extern crate substrate_serializer;
 
+extern crate substrate_codec as codec;
+
 macro_rules! try_opt {
 	($e: expr) => {
 		match $e {

substrate/polkadot/primitives/src/parachain.rs

@@ -19,24 +19,25 @@
 #[cfg(feature = "std")]
 use primitives::bytes;
 use primitives;
+use codec::{Input, Slicable, NonTrivialSlicable};
 use rstd::vec::Vec;
 
 /// Unique identifier of a parachain.
 #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
-pub struct Id(u64);
+pub struct Id(u32);
 
-impl From<Id> for u64 {
+impl From<Id> for u32 {
 	fn from(x: Id) -> Self { x.0 }
 }
 
-impl From<u64> for Id {
+impl From<u32> for Id {
-	fn from(x: u64) -> Self { Id(x) }
+	fn from(x: u32) -> Self { Id(x) }
 }
 
-impl ::codec::Slicable for Id {
+impl Slicable for Id {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		u64::from_slice(value).map(Id)
+		u32::decode(input).map(Id)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -44,6 +45,80 @@ impl ::codec::Slicable for Id {
 	}
 }
 
+/// Identifier for a chain, either one of a number of parachains or the relay chain.
+#[derive(Copy, Clone, PartialEq)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub enum Chain {
+	/// The relay chain.
+	Relay,
+	/// A parachain of the given index.
+	Parachain(Id),
+}
+
+impl Slicable for Chain {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		let disc = try_opt!(u8::decode(input));
+
+		match disc {
+			0 => Some(Chain::Relay),
+			1 => Some(Chain::Parachain(try_opt!(Slicable::decode(input)))),
+			_ => None,
+		}
+	}
+
+	fn to_vec(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+		match *self {
+			Chain::Relay => { 0u8.as_slice_then(|s| v.extend(s)); }
+			Chain::Parachain(id) => {
+				1u8.as_slice_then(|s| v.extend(s));
+				id.as_slice_then(|s| v.extend(s));
+			}
+		}
+
+		v
+	}
+
+	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		f(&self.to_vec().as_slice())
+	}
+}
+
+impl NonTrivialSlicable for Chain { }
+
+/// The duty roster specifying what jobs each validator must do.
+#[derive(Clone, PartialEq)]
+#[cfg_attr(feature = "std", derive(Default, Debug))]
+pub struct DutyRoster {
+	/// Lookup from validator index to chain on which that validator has a duty to validate.
+	pub validator_duty: Vec<Chain>,
+	/// Lookup from validator index to chain on which that validator has a duty to guarantee
+	/// availability.
+	pub guarantor_duty: Vec<Chain>,
+}
+
+impl Slicable for DutyRoster {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(DutyRoster {
+			validator_duty: try_opt!(Slicable::decode(input)),
+			guarantor_duty: try_opt!(Slicable::decode(input)),
+		})
+	}
+
+	fn to_vec(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		v.extend(self.validator_duty.to_vec());
+		v.extend(self.guarantor_duty.to_vec());
+
+		v
+	}
+
+	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		f(&self.to_vec().as_slice())
+	}
+}
+
 /// Candidate parachain block.
 ///
 /// https://github.com/w3f/polkadot-spec/blob/master/spec.md#candidate-para-chain-block
@@ -124,9 +199,9 @@ pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
 pub struct Activity(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
-impl ::codec::Slicable for Activity {
+impl Slicable for Activity {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Activity)
+		Vec::<u8>::decode(input).map(Activity)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {

substrate/polkadot/primitives/src/transaction.rs

@@ -17,7 +17,7 @@
 //! Transaction type.
 
 use rstd::vec::Vec;
-use codec::Slicable;
+use codec::{Input, Slicable};
 
 #[cfg(feature = "std")]
 use std::fmt;
@@ -91,23 +91,23 @@ pub enum Proposal {
 }
 
 impl Slicable for Proposal {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		let id = try_opt!(u8::from_slice(value).and_then(InternalFunctionId::from_u8));
+		let id = try_opt!(u8::decode(input).and_then(InternalFunctionId::from_u8));
 		let function = match id {
 			InternalFunctionId::SystemSetCode =>
-				Proposal::SystemSetCode(try_opt!(Slicable::from_slice(value))),
+				Proposal::SystemSetCode(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::SessionSetLength =>
-				Proposal::SessionSetLength(try_opt!(Slicable::from_slice(value))),
+				Proposal::SessionSetLength(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::SessionForceNewSession => Proposal::SessionForceNewSession,
 			InternalFunctionId::StakingSetSessionsPerEra =>
-				Proposal::StakingSetSessionsPerEra(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetSessionsPerEra(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingSetBondingDuration =>
-				Proposal::StakingSetBondingDuration(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetBondingDuration(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingSetValidatorCount =>
-				Proposal::StakingSetValidatorCount(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetValidatorCount(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingForceNewEra => Proposal::StakingForceNewEra,
 			InternalFunctionId::GovernanceSetApprovalPpmRequired =>
-				Proposal::GovernanceSetApprovalPpmRequired(try_opt!(Slicable::from_slice(value))),
+				Proposal::GovernanceSetApprovalPpmRequired(try_opt!(Slicable::decode(input))),
 		};
 
 		Some(function)
@@ -210,25 +210,25 @@ pub enum Function {
 }
 
 impl Slicable for Function {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		let id = try_opt!(u8::from_slice(value).and_then(FunctionId::from_u8));
+		let id = try_opt!(u8::decode(input).and_then(FunctionId::from_u8));
 		Some(match id {
 			FunctionId::TimestampSet =>
-				Function::TimestampSet(try_opt!(Slicable::from_slice(value))),
+				Function::TimestampSet(try_opt!(Slicable::decode(input))),
 			FunctionId::SessionSetKey =>
-				Function::SessionSetKey(try_opt!(Slicable::from_slice(value))),
+				Function::SessionSetKey(try_opt!(Slicable::decode(input))),
 			FunctionId::StakingStake => Function::StakingStake,
 			FunctionId::StakingUnstake => Function::StakingUnstake,
 			FunctionId::StakingTransfer => {
-				let to  = try_opt!(Slicable::from_slice(value));
+				let to  = try_opt!(Slicable::decode(input));
-				let amount = try_opt!(Slicable::from_slice(value));
+				let amount = try_opt!(Slicable::decode(input));
 
 				Function::StakingTransfer(to, amount)
 			}
 			FunctionId::GovernancePropose =>
-				Function::GovernancePropose(try_opt!(Slicable::from_slice(value))),
+				Function::GovernancePropose(try_opt!(Slicable::decode(input))),
 			FunctionId::GovernanceApprove =>
-				Function::GovernanceApprove(try_opt!(Slicable::from_slice(value))),
+				Function::GovernanceApprove(try_opt!(Slicable::decode(input))),
 		})
 	}
 
@@ -285,11 +285,11 @@ pub struct Transaction {
 }
 
 impl Slicable for Transaction {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Transaction {
-			signed: try_opt!(Slicable::from_slice(value)),
+			signed: try_opt!(Slicable::decode(input)),
-			nonce: try_opt!(Slicable::from_slice(value)),
+			nonce: try_opt!(Slicable::decode(input)),
-			function: try_opt!(Slicable::from_slice(value)),
+			function: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -321,16 +321,16 @@ pub struct UncheckedTransaction {
 }
 
 impl Slicable for UncheckedTransaction {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		// This is a little more complicated than usua since the binary format must be compatible
 		// with substrate's generic `Vec<u8>` type. Basically this just means accepting that there
 		// will be a prefix of u32, which has the total number of bytes following (we don't need
 		// to use this).
-		let _length_do_not_remove_me_see_above: u32 = try_opt!(Slicable::from_slice(value));
+		let _length_do_not_remove_me_see_above: u32 = try_opt!(Slicable::decode(input));
 
 		Some(UncheckedTransaction {
-			transaction: try_opt!(Slicable::from_slice(value)),
+			transaction: try_opt!(Slicable::decode(input)),
-			signature: try_opt!(Slicable::from_slice(value)),
+			signature: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -398,6 +398,6 @@ mod tests {
 
 		let v = Slicable::to_vec(&tx);
 		println!("{}", HexDisplay::from(&v));
-		assert_eq!(UncheckedTransaction::from_slice(&mut &v[..]).unwrap(), tx);
+		assert_eq!(UncheckedTransaction::decode(&mut &v[..]).unwrap(), tx);
 	}
 }

substrate/polkadot/runtime/src/api.rs

@@ -0,0 +1,27 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use runtime::{system, parachains, consensus, session};
+
+impl_stubs!(
+	execute_block => |block| system::internal::execute_block(block),
+	execute_transaction => |(header, utx)| system::internal::execute_transaction(utx, header),
+	finalise_block => |header| system::internal::finalise_block(header),
+	validator_count => |()| session::validator_count(),
+	validators => |()| session::validators(),
+	authorities => |()| consensus::authorities(),
+	duty_roster => |()| parachains::calculate_duty_roster()
+);

substrate/polkadot/runtime/src/genesismap.rs

@@ -16,12 +16,12 @@
 
 //! Tool for creating the genesis block.
 
+use codec::{KeyedVec, Joiner};
+use polkadot_primitives::{BlockNumber, Block, AccountId};
 use std::collections::HashMap;
 use runtime_io::twox_128;
-use codec::{KeyedVec, Joiner};
-use support::Hashable;
-use polkadot_primitives::{BlockNumber, Block, AccountId};
 use runtime::staking::Balance;
+use support::Hashable;
 
 /// Configuration of a general Polkadot genesis block.
 pub struct GenesisConfig {

substrate/polkadot/runtime/src/lib.rs

@@ -37,14 +37,11 @@ extern crate hex_literal;
 #[macro_use]
 pub mod support;
 pub mod runtime;
+pub mod api;
 
 #[cfg(feature = "std")]
 pub mod genesismap;
 
-use rstd::prelude::*;
-use codec::Slicable;
-use polkadot_primitives::{Header, Block, UncheckedTransaction};
-
 /// Type definitions and helpers for transactions.
 pub mod transaction {
 	use rstd::ops;
@@ -83,39 +80,3 @@ pub mod transaction {
 		}
 	}
 }
-
-/// Execute a block, with `input` being the canonical serialisation of the block. Returns the
-/// empty vector.
-pub fn execute_block(mut input: &[u8]) -> Vec<u8> {
-	runtime::system::internal::execute_block(Block::from_slice(&mut input).unwrap());
-	Vec::new()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn execute_transaction(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let utx = UncheckedTransaction::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::execute_transaction(utx, header);
-	header.to_vec()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn finalise_block(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::finalise_block(header);
-	header.to_vec()
-}
-
-/// Run whatever tests we have.
-pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
-	use runtime_io::print;
-
-	print("run_tests...");
-	let block = Block::from_slice(&mut input).unwrap();
-	print("deserialised block.");
-	let stxs = block.transactions.iter().map(Slicable::to_vec).collect::<Vec<_>>();
-	print("reserialised transactions.");
-	[stxs.len() as u8].to_vec()
-}
-
-impl_stubs!(execute_block, execute_transaction, finalise_block, run_tests);

substrate/polkadot/runtime/src/runtime/parachains.rs

@@ -17,34 +17,13 @@
 //! Main parachains logic. For now this is just the determination of which validators do what.
 
 use rstd::prelude::*;
-use rstd::mem;
 use codec::{Slicable, Joiner};
-use support::{Hashable, with_env, storage};
 use runtime::session;
+use support::{Hashable, with_env, storage};
+use polkadot_primitives::parachain::{Id, Chain, DutyRoster};
 
 const PARACHAIN_COUNT: &[u8] = b"par:cou";
 
-/// Identifier for a chain, either one of a number of parachains or the relay chain.
-#[derive(Copy, Clone, PartialEq)]
-#[cfg_attr(test, derive(Debug))]
-pub enum Chain {
-	/// The relay chain.
-	Relay,
-	/// A parachain of the given index.
-	Parachain(u32),
-}
-
-/// The duty roster specifying what jobs each validator must do.
-#[derive(Clone, PartialEq)]
-#[cfg_attr(test, derive(Default, Debug))]
-pub struct DutyRoster {
-	/// Lookup from validator index to chain on which that validator has a duty to validate.
-	pub validator_duty: Vec<Chain>,
-	/// Lookup from validator index to chain on which that validator has a duty to guarantee
-	/// availability.
-	pub guarantor_duty: Vec<Chain>,
-}
-
 /// Get the number of parachains registered at present.
 pub fn parachain_count() -> u32 {
 	storage::get_or(PARACHAIN_COUNT, 0)
@@ -57,7 +36,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 	let validators_per_parachain = (validator_count - 1) / parachain_count;
 
 	let mut roles_val = (0..validator_count).map(|i| match i {
-		i if i < parachain_count * validators_per_parachain => Chain::Parachain(i / validators_per_parachain as u32),
+		i if i < parachain_count * validators_per_parachain =>
+			Chain::Parachain(Id::from(i / validators_per_parachain as u32)),
 		_ => Chain::Relay,
 	}).collect::<Vec<_>>();
 	let mut roles_gua = roles_val.clone();
@@ -74,8 +54,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 		let remaining = (validator_count - i) as usize;
 
 		// 4 * 2 32-bit ints per 256-bit seed.
-		let val_index = u32::from_slice(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let val_index = u32::decode(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
-		let gua_index = u32::from_slice(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let gua_index = u32::decode(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
 
 		if offset == 24 {
 			// into the last 8 bytes - rehash to gather new entropy
@@ -115,7 +95,7 @@ mod tests {
 			let check_roster = |duty_roster: &DutyRoster| {
 				assert_eq!(duty_roster.validator_duty.len(), 8);
 				assert_eq!(duty_roster.guarantor_duty.len(), 8);
-				for i in 0..2 {
+				for i in (0..2).map(Id::from) {
 					assert_eq!(duty_roster.validator_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 					assert_eq!(duty_roster.guarantor_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 				}

substrate/polkadot/runtime/src/runtime/session.rs

@@ -35,7 +35,7 @@ impl StorageVec for ValidatorStorageVec {
 	const PREFIX: &'static[u8] = b"ses:val:";
 }
 
-/// Get the current set of authorities. These are the session keys.
+/// Get the current set of validators.
 pub fn validators() -> Vec<AccountId> {
 	ValidatorStorageVec::items()
 }

substrate/polkadot/runtime/src/support/storage.rs

@@ -18,14 +18,33 @@
 
 use rstd::prelude::*;
 use runtime_io::{self, twox_128};
-use codec::{Slicable, KeyedVec};
+use codec::{Input, Slicable, KeyedVec};
 
 // TODO: consider using blake256 to avoid possible preimage attack.
 
+struct IncrementalInput<'a> {
+	key: &'a [u8],
+	pos: usize,
+}
+
+impl<'a> Input for IncrementalInput<'a> {
+	fn read(&mut self, into: &mut [u8]) -> usize {
+		let len = runtime_io::read_storage(self.key, into, self.pos);
+		let read = ::rstd::cmp::min(len, into.len());
+		self.pos += read;
+		read
+	}
+}
+
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-	let raw = runtime_io::storage(&twox_128(key)[..]);
+	let key = twox_128(key);
-	Slicable::from_slice(&mut &raw[..])
+	let mut input = IncrementalInput {
+		key: &key[..],
+		pos: 0,
+	};
+
+	Slicable::decode(&mut input)
 }
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
@@ -142,12 +161,16 @@ pub trait StorageVec {
 }
 
 pub mod unhashed {
-	use super::{runtime_io, Slicable, KeyedVec, Vec};
+	use super::{runtime_io, Slicable, KeyedVec, Vec, IncrementalInput};
 
 	/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 	pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-		let raw = runtime_io::storage(key);
+		let mut input = IncrementalInput {
-		T::from_slice(&mut &raw[..])
+			key,
+			pos: 0,
+		};
+
+		T::decode(&mut input)
 	}
 
 	/// Return the value of the item in storage under `key`, or the type's default if there is no

substrate/substrate/client/src/lib.rs

@@ -1,20 +1,20 @@
 // Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
+// This file is part of Substrate.
 
-// Polkadot is free software: you can redistribute it and/or modify
+// Substrate is free software: you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 
-// Polkadot is distributed in the hope that it will be useful,
+// Substrate is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU General Public License for more details.
 
 // You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Polkadot Client
+//! Substrate Client
 
 #![warn(missing_docs)]
 
@@ -139,8 +139,9 @@ impl<B, E> Client<B, E> where
 		})
 	}
 
-	fn state_at(&self, id: BlockId) -> error::Result<B::State> {
+	/// Get a reference to the state at a given block.
-		self.backend.state_at(id)
+	pub fn state_at(&self, block: &BlockId) -> error::Result<B::State> {
+		self.backend.state_at(*block)
 	}
 
 	/// Expose backend reference. To be used in tests only
@@ -148,28 +149,33 @@ impl<B, E> Client<B, E> where
 		&self.backend
 	}
 
-	/// Return single storage entry of contract under given address in state in a block of given id.
+	/// Return single storage entry of contract under given address in state in a block of given hash.
 	pub fn storage(&self, id: &BlockId, key: &StorageKey) -> error::Result<StorageData> {
-		Ok(self.state_at(*id)?
+		Ok(self.state_at(id)?
 			.storage(&key.0)
 			.map(|x| StorageData(x.to_vec()))?)
 	}
 
-	/// Execute a call to a contract on top of state in a block of given id.
+	/// Get the code at a given block.
+	pub fn code_at(&self, id: &BlockId) -> error::Result<Vec<u8>> {
+		self.storage(id, &StorageKey(b":code:".to_vec())).map(|data| data.0)
+	}
+
+	/// Execute a call to a contract on top of state in a block of given hash.
 	///
 	/// No changes are made.
 	pub fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<CallResult> {
-		let state = self.state_at(*id)?;
 		let mut changes = state_machine::OverlayedChanges::default();
+		let state = self.state_at(id)?;
 
-		let _ = state_machine::execute(
+		let return_data = state_machine::execute(
 			&state,
 			&mut changes,
 			&self.executor,
 			method,
 			call_data,
 		)?;
-		Ok(CallResult { return_data: vec![], changes })
+		Ok(CallResult { return_data, changes })
 	}
 
 	/// Queue a block for import.

substrate/substrate/codec/src/lib.rs

@@ -20,6 +20,7 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 #![cfg_attr(not(feature = "std"), feature(alloc))]
 
+#[cfg_attr(not(feature = "std"), macro_use)]
 extern crate substrate_runtime_std as rstd;
 
 mod endiansensitive;
@@ -28,6 +29,6 @@ mod joiner;
 mod keyedvec;
 
 pub use self::endiansensitive::EndianSensitive;
-pub use self::slicable::{Slicable, NonTrivialSlicable};
+pub use self::slicable::{Input, Slicable, NonTrivialSlicable};
 pub use self::joiner::Joiner;
 pub use self::keyedvec::KeyedVec;

substrate/substrate/codec/src/slicable.rs

@@ -21,13 +21,25 @@ use rstd::vec::Vec;
 use super::joiner::Joiner;
 use super::endiansensitive::EndianSensitive;
 
+/// Trait that allows reading of data into a slice.
+pub trait Input {
+	/// Read into the provided input slice. Returns the number of bytes read.
+	fn read(&mut self, into: &mut [u8]) -> usize;
+}
+
+impl<'a> Input for &'a [u8] {
+	fn read(&mut self, into: &mut [u8]) -> usize {
+		let len = ::rstd::cmp::min(into.len(), self.len());
+		into[..len].copy_from_slice(&self[..len]);
+		*self = &self[len..];
+		len
+	}
+}
+
 /// Trait that allows zero-copy read/write of value-references to/from slices in LE format.
 pub trait Slicable: Sized {
-	/// Attempt to deserialise the value from a slice. Ignore trailing bytes and
+	/// Attempt to deserialise the value from input.
-	/// set the slice's start to just after the last byte consumed.
+	fn decode<I: Input>(value: &mut I) -> Option<Self>;
-	///
-	/// If `None` is returned, then the slice should be unmodified.
-	fn from_slice(value: &mut &[u8]) -> Option<Self>;
 	/// Convert self to an owned vector.
 	fn to_vec(&self) -> Vec<u8> {
 		self.as_slice_then(|s| s.to_vec())
@@ -40,16 +52,19 @@ pub trait Slicable: Sized {
 pub trait NonTrivialSlicable: Slicable {}
 
 impl<T: EndianSensitive> Slicable for T {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		let size = mem::size_of::<T>();
 		assert!(size > 0, "EndianSensitive can never be implemented for a zero-sized type.");
-		if value.len() >= size {
+		let mut val: T = unsafe { mem::zeroed() };
-			let x: T = unsafe { ::rstd::ptr::read(value.as_ptr() as *const T) };
+
-			*value = &value[size..];
+		unsafe {
-			Some(x.from_le())
+			let raw: &mut [u8] = slice::from_raw_parts_mut(
-		} else {
+				&mut val as *mut T as *mut u8,
-			None
+				size
+			);
+			if input.read(raw) != size { return None }
 		}
+		Some(val.from_le())
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -70,12 +85,15 @@ impl<T: EndianSensitive> Slicable for T {
 }
 
 impl Slicable for Vec<u8> {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		u32::from_slice(value).map(move |len| {
+		u32::decode(input).and_then(move |len| {
 			let len = len as usize;
-			let res = value[..len].to_vec();
+			let mut vec = vec![0; len];
-			*value = &value[len..];
+			if input.read(&mut vec[..len]) != len {
-			res
+				None
+			} else {
+				Some(vec)
+			}
 		})
 	}
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -92,18 +110,60 @@ impl Slicable for Vec<u8> {
 	}
 }
 
+macro_rules! impl_vec_simple_array {
+	($($size:expr),*) => {
+		$(
+			impl<T> Slicable for Vec<[T; $size]>
+				where [T; $size]: EndianSensitive
+			{
+				fn decode<I: Input>(input: &mut I) -> Option<Self> {
+					u32::decode(input).and_then(move |len| {
+						let mut r = Vec::with_capacity(len as usize);
+						for _ in 0..len {
+							r.push(match Slicable::decode(input) {
+								Some(x) => x,
+								None => return None,
+							});
+						}
+
+						Some(r)
+					})
+				}
+
+				fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+					f(&self.to_vec())
+				}
+
+				fn to_vec(&self) -> Vec<u8> {
+					use rstd::iter::Extend;
+
+					let len = self.len();
+					assert!(len <= u32::max_value() as usize, "Attempted to serialize vec with too many elements.");
+
+					let mut r: Vec<u8> = Vec::new().join(&(len as u32));
+					for item in self {
+						r.extend(item.to_vec());
+					}
+					r
+				}
+			}
+		)*
+	}
+}
+
+impl_vec_simple_array!(1, 2, 4, 8, 16, 32, 64);
+
 impl<T: Slicable> NonTrivialSlicable for Vec<T> where Vec<T>: Slicable {}
 
 impl<T: NonTrivialSlicable> Slicable for Vec<T> {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		u32::from_slice(value).and_then(move |len| {
+		u32::decode(input).and_then(move |len| {
-			let len = len as usize;
+			let mut r = Vec::with_capacity(len as usize);
-			let mut r = Vec::with_capacity(len);
 			for _ in 0..len {
-				match T::from_slice(value) {
+				r.push(match T::decode(input) {
+					Some(x) => x,
 					None => return None,
-					Some(v) => r.push(v),
+				});
-				}
 			}
 
 			Some(r)
@@ -128,6 +188,82 @@ impl<T: NonTrivialSlicable> Slicable for Vec<T> {
 	}
 }
 
+impl Slicable for () {
+	fn decode<I: Input>(_: &mut I) -> Option<()> {
+		Some(())
+	}
+
+	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		f(&[])
+	}
+
+	fn to_vec(&self) -> Vec<u8> {
+		Vec::new()
+	}
+}
+
+macro_rules! tuple_impl {
+	($one:ident,) => {
+		impl<$one: Slicable> Slicable for ($one,) {
+			fn decode<I: Input>(input: &mut I) -> Option<Self> {
+				match $one::decode(input) {
+					None => None,
+					Some($one) => Some(($one,)),
+				}
+			}
+
+			fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+				self.0.as_slice_then(f)
+			}
+		}
+	};
+	($first:ident, $($rest:ident,)+) => {
+		impl<$first: Slicable, $($rest: Slicable),+>
+		Slicable for
+		($first, $($rest),+) {
+			fn decode<INPUT: Input>(input: &mut INPUT) -> Option<Self> {
+				Some((
+					match $first::decode(input) {
+						Some(x) => x,
+						None => return None,
+					},
+					$(match $rest::decode(input) {
+						Some(x) => x,
+						None => return None,
+					},)+
+				))
+			}
+
+			fn as_slice_then<RETURN, PROCESS>(&self, f: PROCESS) -> RETURN
+				where PROCESS: FnOnce(&[u8]) -> RETURN
+			{
+				let mut v = Vec::new();
+
+				let (
+					ref $first,
+					$(ref $rest),+
+				) = *self;
+
+				$first.as_slice_then(|s| v.extend(s));
+				$($rest.as_slice_then(|s| v.extend(s));)+
+
+				f(v.as_slice())
+			}
+		}
+
+		tuple_impl!($($rest,)+);
+	}
+}
+
+#[allow(non_snake_case)]
+mod inner_tuple_impl {
+	use rstd::vec::Vec;
+
+	use super::{Input, Slicable};
+	tuple_impl!(A, B, C, D, E, F, G, H, I, J, K,);
+}
+
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/substrate/executor/src/lib.rs

@@ -61,4 +61,4 @@ mod native_executor;
 
 pub mod error;
 pub use wasm_executor::WasmExecutor;
-pub use native_executor::{NativeExecutionDispatch, NativeExecutor};
+pub use native_executor::{with_native_environment, NativeExecutor, NativeExecutionDispatch};

substrate/substrate/executor/src/native_executor.rs

@@ -15,11 +15,23 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use error::{Error, ErrorKind, Result};
-use runtime_io;
 use state_machine::{Externalities, CodeExecutor};
 use wasm_executor::WasmExecutor;
 
-use std::panic::catch_unwind;
+fn safe_call<F, U>(f: F) -> Result<U>
+	where F: ::std::panic::UnwindSafe + FnOnce() -> U
+{
+	::std::panic::catch_unwind(f).map_err(|_| ErrorKind::Runtime.into())
+}
+
+/// Set up the externalities and safe calling environment to execute calls to a native runtime.
+///
+/// If the inner closure panics, it will be caught and return an error.
+pub fn with_native_environment<F, U>(ext: &mut Externalities, f: F) -> Result<U>
+	where F: ::std::panic::UnwindSafe + FnOnce() -> U
+{
+	::runtime_io::with_externalities(ext, move || safe_call(f))
+}
 
 /// Delegate for dispatching a CodeExecutor call to native code.
 pub trait NativeExecutionDispatch {
@@ -28,7 +40,7 @@ pub trait NativeExecutionDispatch {
 
 	/// Dispatch a method and input data to be executed natively. Returns `Some` result or `None`
 	/// if the `method` is unknown. Panics if there's an unrecoverable error.
-	fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>>;
+	fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>>;
 }
 
 /// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence
@@ -38,10 +50,6 @@ pub struct NativeExecutor<D: NativeExecutionDispatch + Sync + Send> {
 	pub _dummy: ::std::marker::PhantomData<D>,
 }
 
-fn safe_call<F: ::std::panic::UnwindSafe + FnOnce() -> Option<Vec<u8>>>(f: F) -> Result<Option<Vec<u8>>> {
-	catch_unwind(f).map_err(|_| ErrorKind::Runtime.into())
-}
-
 impl<D: NativeExecutionDispatch + Sync + Send> CodeExecutor for NativeExecutor<D> {
 	type Error = Error;
 
@@ -53,8 +61,8 @@ impl<D: NativeExecutionDispatch + Sync + Send> CodeExecutor for NativeExecutor<D
 		data: &[u8],
 	) -> Result<Vec<u8>> {
 		if code == D::native_equivalent() {
-			runtime_io::with_externalities(ext, || safe_call(|| D::dispatch(method, data)))?
+			// call native
-				.ok_or(ErrorKind::MethodNotFound(method.to_owned()).into())
+			D::dispatch(ext, method, data)
 		} else {
 			// call into wasm.
 			WasmExecutor.call(ext, code, method, data)

substrate/substrate/primitives/src/block.rs

@@ -19,7 +19,7 @@
 #[cfg(feature = "std")]
 use bytes;
 use rstd::vec::Vec;
-use codec::Slicable;
+use codec::{Input, Slicable};
 use hash::H256;
 
 /// Used to refer to a block number.
@@ -37,8 +37,8 @@ pub type TransactionHash = H256;
 pub struct Transaction(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
 impl Slicable for Transaction {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Transaction)
+		Vec::<u8>::decode(input).map(Transaction)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -54,8 +54,8 @@ impl ::codec::NonTrivialSlicable for Transaction { }
 pub struct Log(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
 impl Slicable for Log {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Log)
+		Vec::<u8>::decode(input).map(Log)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -74,8 +74,8 @@ pub struct Digest {
 }
 
 impl Slicable for Digest {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
-		Vec::<Log>::from_slice(value).map(|logs| Digest { logs })
+		Vec::<Log>::decode(input).map(|logs| Digest { logs })
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -97,10 +97,10 @@ pub struct Block {
 }
 
 impl Slicable for Block {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Block {
-			header: try_opt!(Slicable::from_slice(value)),
+			header: try_opt!(Slicable::decode(input)),
-			transactions: try_opt!(Slicable::from_slice(value)),
+			transactions: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -152,13 +152,13 @@ impl Header {
 }
 
 impl Slicable for Header {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Header {
-			parent_hash: try_opt!(Slicable::from_slice(value)),
+			parent_hash: try_opt!(Slicable::decode(input)),
-			number: try_opt!(Slicable::from_slice(value)),
+			number: try_opt!(Slicable::decode(input)),
-			state_root: try_opt!(Slicable::from_slice(value)),
+			state_root: try_opt!(Slicable::decode(input)),
-			transaction_root: try_opt!(Slicable::from_slice(value)),
+			transaction_root: try_opt!(Slicable::decode(input)),
-			digest: try_opt!(Slicable::from_slice(value)),
+			digest: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -208,6 +208,6 @@ mod tests {
 }"#);
 
 		let v = header.to_vec();
-		assert_eq!(Header::from_slice(&mut &v[..]).unwrap(), header);
+		assert_eq!(Header::decode(&mut &v[..]).unwrap(), header);
 	}
 }

substrate/substrate/primitives/src/hash.rs

@@ -40,8 +40,8 @@ macro_rules! impl_rest {
 		}
 
 		impl ::codec::Slicable for $name {
-			fn from_slice(value: &mut &[u8]) -> Option<Self> {
+			fn decode<I: ::codec::Input>(input: &mut I) -> Option<Self> {
-				<[u8; $len] as ::codec::Slicable>::from_slice(value).map($name)
+				<[u8; $len] as ::codec::Slicable>::decode(input).map($name)
 			}
 
 			fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {

substrate/substrate/runtime-io/with_std.rs

@@ -22,6 +22,8 @@ extern crate substrate_primitives as primitives;
 extern crate triehash;
 extern crate ed25519;
 
+#[doc(hidden)]
+pub extern crate substrate_codec as codec;
 // re-export hashing functions.
 pub use primitives::{blake2_256, twox_128, twox_256};
 
@@ -40,7 +42,7 @@ pub fn storage(key: &[u8]) -> Vec<u8> {
 }
 
 /// Get `key` from storage, placing the value into `value_out` (as much as possible) and return
-/// the number of bytes that the key in storage was.
+/// the number of bytes that the key in storage was beyond the offset.
 pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> usize {
 	ext::with(|ext| {
 		if let Ok(value) = ext.storage(key) {
@@ -124,11 +126,20 @@ pub fn print<T: Printable + Sized>(value: T) {
 
 #[macro_export]
 macro_rules! impl_stubs {
-	($( $name:ident ),*) => {
+	( $( $name:ident => $invoke:expr ),* ) => {
-		pub fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
+		/// Dispatch logic for the native runtime.
+		pub fn dispatch(method: &str, mut data: &[u8]) -> Option<Vec<u8>> {
 			match method {
 				$(
-					stringify!($name) => Some($name(data)),
+					stringify!($name) => {
+						let input = match $crate::codec::Slicable::decode(&mut data) {
+							Some(input) => input,
+							None => panic!("Bad input data provided to {}", stringify!($name)),
+						};
+
+						let output = $invoke(input);
+						Some($crate::codec::Slicable::to_vec(&output))
+					}
 				)*
 				_ => None,
 			}

substrate/substrate/runtime-io/without_std.rs

@@ -17,6 +17,9 @@
 extern crate substrate_runtime_std as rstd;
 extern crate substrate_primitives as primitives;
 
+#[doc(hidden)]
+pub extern crate substrate_codec as codec;
+
 use rstd::intrinsics;
 use rstd::vec::Vec;
 pub use rstd::{mem, slice};
@@ -68,7 +71,7 @@ pub fn set_storage(key: &[u8], value: &[u8]) {
 }
 
 /// Get `key` from storage, placing the value into `value_out` (as much as possible) and return
-/// the number of bytes that the key in storage was.
+/// the number of bytes that the key in storage was beyond the offset.
 pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> usize {
 	unsafe {
 		ext_get_storage_into(
@@ -178,12 +181,11 @@ pub fn print<T: Printable + Sized>(value: T) {
 
 #[macro_export]
 macro_rules! impl_stubs {
-	( $( $name:ident ),* ) => {
+	( $( $new_name:ident => $invoke:expr ),* ) => {
-		pub mod _internal {
 		$(
 			#[no_mangle]
-				pub fn $name(input_data: *mut u8, input_len: usize) -> u64 {
+			pub fn $new_name(input_data: *mut u8, input_len: usize) -> u64 {
-					let input = if input_len == 0 {
+				let mut input = if input_len == 0 {
 					&[0u8; 0]
 				} else {
 					unsafe {
@@ -191,12 +193,21 @@ macro_rules! impl_stubs {
 					}
 				};
 
-					let output = super::$name(input);
+				let input = match $crate::codec::Slicable::decode(&mut input) {
-					let r = output.as_ptr() as u64 + ((output.len() as u64) << 32);
+					Some(input) => input,
-					$crate::mem::forget(output);
+					None => panic!("Bad input data provided to {}", stringify!($name)),
-					r
+				};
+
+				let output = ($invoke)(input);
+				let output = $crate::codec::Slicable::to_vec(&output);
+				let res = output.as_ptr() as u64 + ((output.len() as u64) << 32);
+
+				// Leak the output vector to avoid it being freed.
+				// This is fine in a WASM context since the heap
+				// will be discarded after the call.
+				::core::mem::forget(output);
+				res
 			}
 		)*
 	}
 }
-}

substrate/substrate/runtime-std/src/lib.rs

@@ -21,6 +21,7 @@
 #![cfg_attr(not(feature = "std"), feature(lang_items))]
 #![cfg_attr(not(feature = "std"), feature(core_intrinsics))]
 #![cfg_attr(not(feature = "std"), feature(alloc))]
+#![cfg_attr(not(feature = "std"), feature(macro_reexport))]
 
 #![cfg_attr(feature = "std", doc = "Polkadot runtime standard library as compiled when linked with Rust's standard library.")]
 #![cfg_attr(not(feature = "std"), doc = "Polkadot's runtime standard library as compiled without Rust's standard library.")]

substrate/substrate/runtime-std/with_std.rs

@@ -14,12 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-pub use std::vec;
-pub use std::rc;
-pub use std::cell;
 pub use std::boxed;
-pub use std::slice;
+pub use std::cell;
+pub use std::cmp;
+pub use std::iter;
 pub use std::mem;
 pub use std::ops;
-pub use std::iter;
 pub use std::ptr;
+pub use std::rc;
+pub use std::slice;
+pub use std::vec;

substrate/substrate/runtime-std/without_std.rs

@@ -15,19 +15,21 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 #[cfg(feature = "nightly")]
+#[macro_reexport(vec)]
 extern crate alloc;
 #[cfg(feature = "nightly")]
 extern crate pwasm_libc;
 #[cfg(feature = "nightly")]
 extern crate pwasm_alloc;
 
-pub use alloc::vec;
 pub use alloc::boxed;
 pub use alloc::rc;
-pub use core::mem;
+pub use alloc::vec;
-pub use core::slice;
 pub use core::cell;
-pub use core::ops;
+pub use core::cmp;
-pub use core::iter;
-pub use core::ptr;
 pub use core::intrinsics;
+pub use core::iter;
+pub use core::mem;
+pub use core::ops;
+pub use core::ptr;
+pub use core::slice;