Re-introduce zero copy codec and add minimal polkadot client API which uses linked native runtime (#65)

* client-api type and move duty roster types to primitives

* tuple implementation for slicable

* mild cleanup of deserialization code

* stubs which handle encoding and decoding themselves

* fancier impl_stubs macro

* zero-copy slicable API

* minimal polkadot-client API

* fix WASM API generation

* move native environment stuff to substrate executor

* fix warnings and grumbles

polkadot/api/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "polkadot-api"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+error-chain = "0.11"
+polkadot-executor = { path = "../executor" }
+polkadot-runtime = { path = "../runtime" }
+polkadot-primitives = { path = "../primitives" }
+substrate-client = { path = "../../substrate/client" }
+substrate-executor = { path = "../../substrate/executor" }
+substrate-state-machine = { path = "../../substrate/state-machine" }

polkadot/api/src/lib.rs

@@ -0,0 +1,118 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Strongly typed API for Polkadot based around the locally-compiled native
+//! runtime.
+
+extern crate polkadot_executor as polkadot_executor;
+extern crate polkadot_runtime ;
+extern crate polkadot_primitives as primitives;
+extern crate substrate_client as client;
+extern crate substrate_executor as substrate_executor;
+extern crate substrate_state_machine as state_machine;
+
+#[macro_use]
+extern crate error_chain;
+
+use client::backend::Backend;
+use client::blockchain::BlockId;
+use client::Client;
+use polkadot_runtime::runtime;
+use polkadot_executor::LocalNativeExecutionDispatch as LocalDispatch;
+use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
+use primitives::{AccountId, SessionKey};
+use primitives::parachain::DutyRoster;
+
+error_chain! {
+	errors {
+		/// Unknown runtime code.
+		UnknownRuntime {
+			description("Unknown runtime code")
+			display("Unknown runtime code")
+		}
+		UnknownBlock(b: BlockId) {
+			description("Unknown block")
+			display("Unknown block")
+		}
+		/// Some other error.
+		// TODO: allow to be specified as associated type of PolkadotApi
+		Other(e: Box<::std::error::Error + Send>) {
+			description("Other error")
+			display("Other error: {}", e.description())
+		}
+	}
+
+	links {
+		Executor(substrate_executor::error::Error, substrate_executor::error::ErrorKind);
+	}
+}
+
+/// Trait encapsulating the Polkadot API.
+///
+/// All calls should fail when the exact runtime is unknown.
+pub trait PolkadotApi {
+	/// Get authorities at a given block.
+	fn authorities(&self, at: &BlockId) -> Result<Vec<SessionKey>>;
+
+	/// Get validators at a given block.
+	fn validators(&self, at: &BlockId) -> Result<Vec<AccountId>>;
+
+	/// Get the authority duty roster at a block.
+	fn duty_roster(&self, at: &BlockId) -> Result<DutyRoster>;
+}
+
+fn convert_client_error(e: client::error::Error) -> Error {
+	match e {
+		client::error::Error(client::error::ErrorKind::UnknownBlock(b), _) => Error::from_kind(ErrorKind::UnknownBlock(b)),
+		other => Error::from_kind(ErrorKind::Other(Box::new(other) as Box<_>)),
+	}
+}
+
+// set up the necessary scaffolding to execute the runtime.
+macro_rules! with_runtime {
+	($client: ident, $at: expr, $exec: expr) => {{
+		// bail if the code is not the same as the natively linked.
+		if $client.code_at($at).map_err(convert_client_error)? != LocalDispatch::native_equivalent() {
+			bail!(ErrorKind::UnknownRuntime);
+		}
+
+		$client.state_at($at).map_err(convert_client_error).and_then(|state| {
+			let mut changes = Default::default();
+			let mut ext = state_machine::Ext {
+				overlay: &mut changes,
+				backend: &state,
+			};
+
+			::substrate_executor::with_native_environment(&mut ext, $exec).map_err(Into::into)
+		})
+	}}
+}
+
+impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
+	where ::client::error::Error: From<<<B as Backend>::State as state_machine::backend::Backend>::Error>
+{
+	fn authorities(&self, at: &BlockId) -> Result<Vec<SessionKey>> {
+		with_runtime!(self, at, ::runtime::consensus::authorities)
+	}
+
+	fn validators(&self, at: &BlockId) -> Result<Vec<AccountId>> {
+		with_runtime!(self, at, ::runtime::session::validators)
+	}
+
+	fn duty_roster(&self, at: &BlockId) -> Result<DutyRoster> {
+		with_runtime!(self, at, ::runtime::parachains::calculate_duty_roster)
+	}
+}

polkadot/cli/src/genesis.rs

@@ -89,7 +89,7 @@ mod tests {
 				"execute_transaction",
 				&vec![].join(&header).join(tx)
 			).unwrap();
-			header = Header::from_slice(&mut &ret_data[..]).unwrap();
+			header = Header::decode(&mut &ret_data[..]).unwrap();
 		}
 
 		let ret_data = execute(
@@ -99,7 +99,7 @@ mod tests {
 			"finalise_block",
 			&vec![].join(&header)
 		).unwrap();
-		header = Header::from_slice(&mut &ret_data[..]).unwrap();
+		header = Header::decode(&mut &ret_data[..]).unwrap();
 
 		(vec![].join(&Block { header, transactions }), hash.into())
 	}

polkadot/cli/src/lib.rs

@@ -83,7 +83,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 		storage = genesis_config.genesis_map();
 		let block = genesis::construct_genesis_block(&storage);
 		storage.extend(additional_storage_with_genesis(&block));
-		(primitives::block::Header::from_slice(&mut block.header.to_vec().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+		(primitives::block::Header::decode(&mut block.header.to_vec().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
 	};
 	let client = client::new_in_mem(executor, prepare_genesis)?;
 

polkadot/executor/src/lib.rs

@@ -31,7 +31,9 @@ extern crate triehash;
 extern crate hex_literal;
 
 use polkadot_runtime as runtime;
+use substrate_executor::error::{Error, ErrorKind};
 use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
+use state_machine::Externalities;
 
 /// A null struct which implements `NativeExecutionDispatch` feeding in the hard-coded runtime.
 pub struct LocalNativeExecutionDispatch;
@@ -43,8 +45,9 @@ impl NativeExecutionDispatch for LocalNativeExecutionDispatch {
 		include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm")
 	}
 
-	fn dispatch(method: &str, data: &[u8]) -> Option<Vec<u8>> {
-		runtime::dispatch(method, data)
+	fn dispatch(ext: &mut Externalities, method: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
+		::substrate_executor::with_native_environment(ext, move || runtime::api::dispatch(method, data))?
+			.ok_or_else(|| ErrorKind::MethodNotFound(method.to_owned()).into())
 	}
 }
 
@@ -236,14 +239,6 @@ mod tests {
 		)
 	}
 
-	#[test]
-	fn test_execution_works() {
-		let mut t = new_test_ext();
-		println!("Testing Wasm...");
-		let r = WasmExecutor.call(&mut t, COMPACT_CODE, "run_tests", &block2().0);
-		assert!(r.is_ok());
-	}
-
 	#[test]
 	fn full_native_block_import_works() {
 		let mut t = new_test_ext();

polkadot/primitives/src/block.rs

@@ -20,7 +20,7 @@
 use primitives::bytes;
 use primitives::H256;
 use rstd::vec::Vec;
-use codec::Slicable;
+use codec::{Input, Slicable};
 use transaction::UncheckedTransaction;
 
 /// Used to refer to a block number.
@@ -38,8 +38,8 @@ pub type TransactionHash = H256;
 pub struct Log(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
 impl Slicable for Log {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Log)
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Vec::<u8>::decode(input).map(Log)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -58,8 +58,8 @@ pub struct Digest {
 }
 
 impl Slicable for Digest {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		Vec::<Log>::from_slice(value).map(|logs| Digest { logs })
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Vec::<Log>::decode(input).map(|logs| Digest { logs })
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -81,11 +81,9 @@ pub struct Block {
 }
 
 impl Slicable for Block {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		Some(Block {
-			header: try_opt!(Slicable::from_slice(value)),
-			transactions: try_opt!(Slicable::from_slice(value)),
-		})
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		let (header, transactions) = try_opt!(Slicable::decode(input));
+		Some(Block { header, transactions })
 	}
 
 	fn to_vec(&self) -> Vec<u8> {
@@ -136,13 +134,13 @@ impl Header {
 }
 
 impl Slicable for Header {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Header {
-			parent_hash: try_opt!(Slicable::from_slice(value)),
-			number: try_opt!(Slicable::from_slice(value)),
-			state_root: try_opt!(Slicable::from_slice(value)),
-			transaction_root: try_opt!(Slicable::from_slice(value)),
-			digest: try_opt!(Slicable::from_slice(value)),
+			parent_hash: try_opt!(Slicable::decode(input)),
+			number: try_opt!(Slicable::decode(input)),
+			state_root: try_opt!(Slicable::decode(input)),
+			transaction_root: try_opt!(Slicable::decode(input)),
+			digest: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -192,6 +190,6 @@ mod tests {
 }"#);
 
 		let v = header.to_vec();
-		assert_eq!(Header::from_slice(&mut &v[..]).unwrap(), header);
+		assert_eq!(Header::decode(&mut &v[..]).unwrap(), header);
 	}
 }

polkadot/primitives/src/lib.rs

@@ -29,11 +29,12 @@ extern crate serde_derive;
 extern crate serde;
 
 extern crate substrate_runtime_std as rstd;
-extern crate substrate_codec as codec;
 extern crate substrate_primitives as primitives;
 #[cfg(test)]
 extern crate substrate_serializer;
 
+extern crate substrate_codec as codec;
+
 macro_rules! try_opt {
 	($e: expr) => {
 		match $e {

polkadot/primitives/src/parachain.rs

@@ -19,24 +19,25 @@
 #[cfg(feature = "std")]
 use primitives::bytes;
 use primitives;
+use codec::{Input, Slicable, NonTrivialSlicable};
 use rstd::vec::Vec;
 
 /// Unique identifier of a parachain.
 #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
-pub struct Id(u64);
+pub struct Id(u32);
 
-impl From<Id> for u64 {
+impl From<Id> for u32 {
 	fn from(x: Id) -> Self { x.0 }
 }
 
-impl From<u64> for Id {
-	fn from(x: u64) -> Self { Id(x) }
+impl From<u32> for Id {
+	fn from(x: u32) -> Self { Id(x) }
 }
 
-impl ::codec::Slicable for Id {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		u64::from_slice(value).map(Id)
+impl Slicable for Id {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		u32::decode(input).map(Id)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
@@ -44,6 +45,80 @@ impl ::codec::Slicable for Id {
 	}
 }
 
+/// Identifier for a chain, either one of a number of parachains or the relay chain.
+#[derive(Copy, Clone, PartialEq)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub enum Chain {
+	/// The relay chain.
+	Relay,
+	/// A parachain of the given index.
+	Parachain(Id),
+}
+
+impl Slicable for Chain {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		let disc = try_opt!(u8::decode(input));
+
+		match disc {
+			0 => Some(Chain::Relay),
+			1 => Some(Chain::Parachain(try_opt!(Slicable::decode(input)))),
+			_ => None,
+		}
+	}
+
+	fn to_vec(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+		match *self {
+			Chain::Relay => { 0u8.as_slice_then(|s| v.extend(s)); }
+			Chain::Parachain(id) => {
+				1u8.as_slice_then(|s| v.extend(s));
+				id.as_slice_then(|s| v.extend(s));
+			}
+		}
+
+		v
+	}
+
+	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		f(&self.to_vec().as_slice())
+	}
+}
+
+impl NonTrivialSlicable for Chain { }
+
+/// The duty roster specifying what jobs each validator must do.
+#[derive(Clone, PartialEq)]
+#[cfg_attr(feature = "std", derive(Default, Debug))]
+pub struct DutyRoster {
+	/// Lookup from validator index to chain on which that validator has a duty to validate.
+	pub validator_duty: Vec<Chain>,
+	/// Lookup from validator index to chain on which that validator has a duty to guarantee
+	/// availability.
+	pub guarantor_duty: Vec<Chain>,
+}
+
+impl Slicable for DutyRoster {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(DutyRoster {
+			validator_duty: try_opt!(Slicable::decode(input)),
+			guarantor_duty: try_opt!(Slicable::decode(input)),
+		})
+	}
+
+	fn to_vec(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		v.extend(self.validator_duty.to_vec());
+		v.extend(self.guarantor_duty.to_vec());
+
+		v
+	}
+
+	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		f(&self.to_vec().as_slice())
+	}
+}
+
 /// Candidate parachain block.
 ///
 /// https://github.com/w3f/polkadot-spec/blob/master/spec.md#candidate-para-chain-block
@@ -124,9 +199,9 @@ pub struct ValidationCode(#[cfg_attr(feature = "std", serde(with="bytes"))] pub
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
 pub struct Activity(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
-impl ::codec::Slicable for Activity {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		Vec::<u8>::from_slice(value).map(Activity)
+impl Slicable for Activity {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Vec::<u8>::decode(input).map(Activity)
 	}
 
 	fn as_slice_then<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {

polkadot/primitives/src/transaction.rs

@@ -17,7 +17,7 @@
 //! Transaction type.
 
 use rstd::vec::Vec;
-use codec::Slicable;
+use codec::{Input, Slicable};
 
 #[cfg(feature = "std")]
 use std::fmt;
@@ -91,23 +91,23 @@ pub enum Proposal {
 }
 
 impl Slicable for Proposal {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		let id = try_opt!(u8::from_slice(value).and_then(InternalFunctionId::from_u8));
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		let id = try_opt!(u8::decode(input).and_then(InternalFunctionId::from_u8));
 		let function = match id {
 			InternalFunctionId::SystemSetCode =>
-				Proposal::SystemSetCode(try_opt!(Slicable::from_slice(value))),
+				Proposal::SystemSetCode(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::SessionSetLength =>
-				Proposal::SessionSetLength(try_opt!(Slicable::from_slice(value))),
+				Proposal::SessionSetLength(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::SessionForceNewSession => Proposal::SessionForceNewSession,
 			InternalFunctionId::StakingSetSessionsPerEra =>
-				Proposal::StakingSetSessionsPerEra(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetSessionsPerEra(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingSetBondingDuration =>
-				Proposal::StakingSetBondingDuration(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetBondingDuration(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingSetValidatorCount =>
-				Proposal::StakingSetValidatorCount(try_opt!(Slicable::from_slice(value))),
+				Proposal::StakingSetValidatorCount(try_opt!(Slicable::decode(input))),
 			InternalFunctionId::StakingForceNewEra => Proposal::StakingForceNewEra,
 			InternalFunctionId::GovernanceSetApprovalPpmRequired =>
-				Proposal::GovernanceSetApprovalPpmRequired(try_opt!(Slicable::from_slice(value))),
+				Proposal::GovernanceSetApprovalPpmRequired(try_opt!(Slicable::decode(input))),
 		};
 
 		Some(function)
@@ -210,25 +210,25 @@ pub enum Function {
 }
 
 impl Slicable for Function {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
-		let id = try_opt!(u8::from_slice(value).and_then(FunctionId::from_u8));
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		let id = try_opt!(u8::decode(input).and_then(FunctionId::from_u8));
 		Some(match id {
 			FunctionId::TimestampSet =>
-				Function::TimestampSet(try_opt!(Slicable::from_slice(value))),
+				Function::TimestampSet(try_opt!(Slicable::decode(input))),
 			FunctionId::SessionSetKey =>
-				Function::SessionSetKey(try_opt!(Slicable::from_slice(value))),
+				Function::SessionSetKey(try_opt!(Slicable::decode(input))),
 			FunctionId::StakingStake => Function::StakingStake,
 			FunctionId::StakingUnstake => Function::StakingUnstake,
 			FunctionId::StakingTransfer => {
-				let to  = try_opt!(Slicable::from_slice(value));
-				let amount = try_opt!(Slicable::from_slice(value));
+				let to  = try_opt!(Slicable::decode(input));
+				let amount = try_opt!(Slicable::decode(input));
 
 				Function::StakingTransfer(to, amount)
 			}
 			FunctionId::GovernancePropose =>
-				Function::GovernancePropose(try_opt!(Slicable::from_slice(value))),
+				Function::GovernancePropose(try_opt!(Slicable::decode(input))),
 			FunctionId::GovernanceApprove =>
-				Function::GovernanceApprove(try_opt!(Slicable::from_slice(value))),
+				Function::GovernanceApprove(try_opt!(Slicable::decode(input))),
 		})
 	}
 
@@ -285,11 +285,11 @@ pub struct Transaction {
 }
 
 impl Slicable for Transaction {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		Some(Transaction {
-			signed: try_opt!(Slicable::from_slice(value)),
-			nonce: try_opt!(Slicable::from_slice(value)),
-			function: try_opt!(Slicable::from_slice(value)),
+			signed: try_opt!(Slicable::decode(input)),
+			nonce: try_opt!(Slicable::decode(input)),
+			function: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -321,16 +321,16 @@ pub struct UncheckedTransaction {
 }
 
 impl Slicable for UncheckedTransaction {
-	fn from_slice(value: &mut &[u8]) -> Option<Self> {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		// This is a little more complicated than usua since the binary format must be compatible
 		// with substrate's generic `Vec<u8>` type. Basically this just means accepting that there
 		// will be a prefix of u32, which has the total number of bytes following (we don't need
 		// to use this).
-		let _length_do_not_remove_me_see_above: u32 = try_opt!(Slicable::from_slice(value));
+		let _length_do_not_remove_me_see_above: u32 = try_opt!(Slicable::decode(input));
 
 		Some(UncheckedTransaction {
-			transaction: try_opt!(Slicable::from_slice(value)),
-			signature: try_opt!(Slicable::from_slice(value)),
+			transaction: try_opt!(Slicable::decode(input)),
+			signature: try_opt!(Slicable::decode(input)),
 		})
 	}
 
@@ -398,6 +398,6 @@ mod tests {
 
 		let v = Slicable::to_vec(&tx);
 		println!("{}", HexDisplay::from(&v));
-		assert_eq!(UncheckedTransaction::from_slice(&mut &v[..]).unwrap(), tx);
+		assert_eq!(UncheckedTransaction::decode(&mut &v[..]).unwrap(), tx);
 	}
 }

polkadot/runtime/src/api.rs

@@ -0,0 +1,27 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use runtime::{system, parachains, consensus, session};
+
+impl_stubs!(
+	execute_block => |block| system::internal::execute_block(block),
+	execute_transaction => |(header, utx)| system::internal::execute_transaction(utx, header),
+	finalise_block => |header| system::internal::finalise_block(header),
+	validator_count => |()| session::validator_count(),
+	validators => |()| session::validators(),
+	authorities => |()| consensus::authorities(),
+	duty_roster => |()| parachains::calculate_duty_roster()
+);

polkadot/runtime/src/genesismap.rs

@@ -16,12 +16,12 @@
 
 //! Tool for creating the genesis block.
 
+use codec::{KeyedVec, Joiner};
+use polkadot_primitives::{BlockNumber, Block, AccountId};
 use std::collections::HashMap;
 use runtime_io::twox_128;
-use codec::{KeyedVec, Joiner};
-use support::Hashable;
-use polkadot_primitives::{BlockNumber, Block, AccountId};
 use runtime::staking::Balance;
+use support::Hashable;
 
 /// Configuration of a general Polkadot genesis block.
 pub struct GenesisConfig {

polkadot/runtime/src/lib.rs

@@ -37,14 +37,11 @@ extern crate hex_literal;
 #[macro_use]
 pub mod support;
 pub mod runtime;
+pub mod api;
 
 #[cfg(feature = "std")]
 pub mod genesismap;
 
-use rstd::prelude::*;
-use codec::Slicable;
-use polkadot_primitives::{Header, Block, UncheckedTransaction};
-
 /// Type definitions and helpers for transactions.
 pub mod transaction {
 	use rstd::ops;
@@ -83,39 +80,3 @@ pub mod transaction {
 		}
 	}
 }
-
-/// Execute a block, with `input` being the canonical serialisation of the block. Returns the
-/// empty vector.
-pub fn execute_block(mut input: &[u8]) -> Vec<u8> {
-	runtime::system::internal::execute_block(Block::from_slice(&mut input).unwrap());
-	Vec::new()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn execute_transaction(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let utx = UncheckedTransaction::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::execute_transaction(utx, header);
-	header.to_vec()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn finalise_block(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::finalise_block(header);
-	header.to_vec()
-}
-
-/// Run whatever tests we have.
-pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
-	use runtime_io::print;
-
-	print("run_tests...");
-	let block = Block::from_slice(&mut input).unwrap();
-	print("deserialised block.");
-	let stxs = block.transactions.iter().map(Slicable::to_vec).collect::<Vec<_>>();
-	print("reserialised transactions.");
-	[stxs.len() as u8].to_vec()
-}
-
-impl_stubs!(execute_block, execute_transaction, finalise_block, run_tests);

polkadot/runtime/src/runtime/parachains.rs

@@ -17,34 +17,13 @@
 //! Main parachains logic. For now this is just the determination of which validators do what.
 
 use rstd::prelude::*;
-use rstd::mem;
 use codec::{Slicable, Joiner};
-use support::{Hashable, with_env, storage};
 use runtime::session;
+use support::{Hashable, with_env, storage};
+use polkadot_primitives::parachain::{Id, Chain, DutyRoster};
 
 const PARACHAIN_COUNT: &[u8] = b"par:cou";
 
-/// Identifier for a chain, either one of a number of parachains or the relay chain.
-#[derive(Copy, Clone, PartialEq)]
-#[cfg_attr(test, derive(Debug))]
-pub enum Chain {
-	/// The relay chain.
-	Relay,
-	/// A parachain of the given index.
-	Parachain(u32),
-}
-
-/// The duty roster specifying what jobs each validator must do.
-#[derive(Clone, PartialEq)]
-#[cfg_attr(test, derive(Default, Debug))]
-pub struct DutyRoster {
-	/// Lookup from validator index to chain on which that validator has a duty to validate.
-	pub validator_duty: Vec<Chain>,
-	/// Lookup from validator index to chain on which that validator has a duty to guarantee
-	/// availability.
-	pub guarantor_duty: Vec<Chain>,
-}
-
 /// Get the number of parachains registered at present.
 pub fn parachain_count() -> u32 {
 	storage::get_or(PARACHAIN_COUNT, 0)
@@ -57,7 +36,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 	let validators_per_parachain = (validator_count - 1) / parachain_count;
 
 	let mut roles_val = (0..validator_count).map(|i| match i {
-		i if i < parachain_count * validators_per_parachain => Chain::Parachain(i / validators_per_parachain as u32),
+		i if i < parachain_count * validators_per_parachain =>
+			Chain::Parachain(Id::from(i / validators_per_parachain as u32)),
 		_ => Chain::Relay,
 	}).collect::<Vec<_>>();
 	let mut roles_gua = roles_val.clone();
@@ -74,8 +54,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 		let remaining = (validator_count - i) as usize;
 
 		// 4 * 2 32-bit ints per 256-bit seed.
-		let val_index = u32::from_slice(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
-		let gua_index = u32::from_slice(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let val_index = u32::decode(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let gua_index = u32::decode(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
 
 		if offset == 24 {
 			// into the last 8 bytes - rehash to gather new entropy
@@ -115,7 +95,7 @@ mod tests {
 			let check_roster = |duty_roster: &DutyRoster| {
 				assert_eq!(duty_roster.validator_duty.len(), 8);
 				assert_eq!(duty_roster.guarantor_duty.len(), 8);
-				for i in 0..2 {
+				for i in (0..2).map(Id::from) {
 					assert_eq!(duty_roster.validator_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 					assert_eq!(duty_roster.guarantor_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 				}

polkadot/runtime/src/runtime/session.rs

@@ -35,7 +35,7 @@ impl StorageVec for ValidatorStorageVec {
 	const PREFIX: &'static[u8] = b"ses:val:";
 }
 
-/// Get the current set of authorities. These are the session keys.
+/// Get the current set of validators.
 pub fn validators() -> Vec<AccountId> {
 	ValidatorStorageVec::items()
 }

polkadot/runtime/src/support/storage.rs

@@ -18,14 +18,33 @@
 
 use rstd::prelude::*;
 use runtime_io::{self, twox_128};
-use codec::{Slicable, KeyedVec};
+use codec::{Input, Slicable, KeyedVec};
 
 // TODO: consider using blake256 to avoid possible preimage attack.
 
+struct IncrementalInput<'a> {
+	key: &'a [u8],
+	pos: usize,
+}
+
+impl<'a> Input for IncrementalInput<'a> {
+	fn read(&mut self, into: &mut [u8]) -> usize {
+		let len = runtime_io::read_storage(self.key, into, self.pos);
+		let read = ::rstd::cmp::min(len, into.len());
+		self.pos += read;
+		read
+	}
+}
+
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-	let raw = runtime_io::storage(&twox_128(key)[..]);
-	Slicable::from_slice(&mut &raw[..])
+	let key = twox_128(key);
+	let mut input = IncrementalInput {
+		key: &key[..],
+		pos: 0,
+	};
+
+	Slicable::decode(&mut input)
 }
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
@@ -142,12 +161,16 @@ pub trait StorageVec {
 }
 
 pub mod unhashed {
-	use super::{runtime_io, Slicable, KeyedVec, Vec};
+	use super::{runtime_io, Slicable, KeyedVec, Vec, IncrementalInput};
 
 	/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 	pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-		let raw = runtime_io::storage(key);
-		T::from_slice(&mut &raw[..])
+		let mut input = IncrementalInput {
+			key,
+			pos: 0,
+		};
+
+		T::decode(&mut input)
 	}
 
 	/// Return the value of the item in storage under `key`, or the type's default if there is no