Re-introduce zero copy codec and add minimal polkadot client API which uses linked native runtime (#65)

* client-api type and move duty roster types to primitives

* tuple implementation for slicable

* mild cleanup of deserialization code

* stubs which handle encoding and decoding themselves

* fancier impl_stubs macro

* zero-copy slicable API

* minimal polkadot-client API

* fix WASM API generation

* move native environment stuff to substrate executor

* fix warnings and grumbles

substrate/polkadot/runtime/src/api.rs

@@ -0,0 +1,27 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use runtime::{system, parachains, consensus, session};
+
+impl_stubs!(
+	execute_block => |block| system::internal::execute_block(block),
+	execute_transaction => |(header, utx)| system::internal::execute_transaction(utx, header),
+	finalise_block => |header| system::internal::finalise_block(header),
+	validator_count => |()| session::validator_count(),
+	validators => |()| session::validators(),
+	authorities => |()| consensus::authorities(),
+	duty_roster => |()| parachains::calculate_duty_roster()
+);

substrate/polkadot/runtime/src/genesismap.rs

@@ -16,12 +16,12 @@
 
 //! Tool for creating the genesis block.
 
+use codec::{KeyedVec, Joiner};
+use polkadot_primitives::{BlockNumber, Block, AccountId};
 use std::collections::HashMap;
 use runtime_io::twox_128;
-use codec::{KeyedVec, Joiner};
-use support::Hashable;
-use polkadot_primitives::{BlockNumber, Block, AccountId};
 use runtime::staking::Balance;
+use support::Hashable;
 
 /// Configuration of a general Polkadot genesis block.
 pub struct GenesisConfig {

substrate/polkadot/runtime/src/lib.rs

@@ -37,14 +37,11 @@ extern crate hex_literal;
 #[macro_use]
 pub mod support;
 pub mod runtime;
+pub mod api;
 
 #[cfg(feature = "std")]
 pub mod genesismap;
 
-use rstd::prelude::*;
-use codec::Slicable;
-use polkadot_primitives::{Header, Block, UncheckedTransaction};
-
 /// Type definitions and helpers for transactions.
 pub mod transaction {
 	use rstd::ops;
@@ -83,39 +80,3 @@ pub mod transaction {
 		}
 	}
 }
-
-/// Execute a block, with `input` being the canonical serialisation of the block. Returns the
-/// empty vector.
-pub fn execute_block(mut input: &[u8]) -> Vec<u8> {
-	runtime::system::internal::execute_block(Block::from_slice(&mut input).unwrap());
-	Vec::new()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn execute_transaction(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let utx = UncheckedTransaction::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::execute_transaction(utx, header);
-	header.to_vec()
-}
-
-/// Execute a given, serialised, transaction. Returns the empty vector.
-pub fn finalise_block(mut input: &[u8]) -> Vec<u8> {
-	let header = Header::from_slice(&mut input).unwrap();
-	let header = runtime::system::internal::finalise_block(header);
-	header.to_vec()
-}
-
-/// Run whatever tests we have.
-pub fn run_tests(mut input: &[u8]) -> Vec<u8> {
-	use runtime_io::print;
-
-	print("run_tests...");
-	let block = Block::from_slice(&mut input).unwrap();
-	print("deserialised block.");
-	let stxs = block.transactions.iter().map(Slicable::to_vec).collect::<Vec<_>>();
-	print("reserialised transactions.");
-	[stxs.len() as u8].to_vec()
-}
-
-impl_stubs!(execute_block, execute_transaction, finalise_block, run_tests);

substrate/polkadot/runtime/src/runtime/parachains.rs

@@ -17,34 +17,13 @@
 //! Main parachains logic. For now this is just the determination of which validators do what.
 
 use rstd::prelude::*;
-use rstd::mem;
 use codec::{Slicable, Joiner};
-use support::{Hashable, with_env, storage};
 use runtime::session;
+use support::{Hashable, with_env, storage};
+use polkadot_primitives::parachain::{Id, Chain, DutyRoster};
 
 const PARACHAIN_COUNT: &[u8] = b"par:cou";
 
-/// Identifier for a chain, either one of a number of parachains or the relay chain.
-#[derive(Copy, Clone, PartialEq)]
-#[cfg_attr(test, derive(Debug))]
-pub enum Chain {
-	/// The relay chain.
-	Relay,
-	/// A parachain of the given index.
-	Parachain(u32),
-}
-
-/// The duty roster specifying what jobs each validator must do.
-#[derive(Clone, PartialEq)]
-#[cfg_attr(test, derive(Default, Debug))]
-pub struct DutyRoster {
-	/// Lookup from validator index to chain on which that validator has a duty to validate.
-	pub validator_duty: Vec<Chain>,
-	/// Lookup from validator index to chain on which that validator has a duty to guarantee
-	/// availability.
-	pub guarantor_duty: Vec<Chain>,
-}
-
 /// Get the number of parachains registered at present.
 pub fn parachain_count() -> u32 {
 	storage::get_or(PARACHAIN_COUNT, 0)
@@ -57,7 +36,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 	let validators_per_parachain = (validator_count - 1) / parachain_count;
 
 	let mut roles_val = (0..validator_count).map(|i| match i {
-		i if i < parachain_count * validators_per_parachain => Chain::Parachain(i / validators_per_parachain as u32),
+		i if i < parachain_count * validators_per_parachain =>
+			Chain::Parachain(Id::from(i / validators_per_parachain as u32)),
 		_ => Chain::Relay,
 	}).collect::<Vec<_>>();
 	let mut roles_gua = roles_val.clone();
@@ -74,8 +54,8 @@ pub fn calculate_duty_roster() -> DutyRoster {
 		let remaining = (validator_count - i) as usize;
 
 		// 4 * 2 32-bit ints per 256-bit seed.
-		let val_index = u32::from_slice(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
-		let gua_index = u32::from_slice(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let val_index = u32::decode(&mut &seed[offset..offset + 4]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
+		let gua_index = u32::decode(&mut &seed[offset + 4..offset + 8]).expect("using 4 bytes for a 32-bit quantity") as usize % remaining;
 
 		if offset == 24 {
 			// into the last 8 bytes - rehash to gather new entropy
@@ -115,7 +95,7 @@ mod tests {
 			let check_roster = |duty_roster: &DutyRoster| {
 				assert_eq!(duty_roster.validator_duty.len(), 8);
 				assert_eq!(duty_roster.guarantor_duty.len(), 8);
-				for i in 0..2 {
+				for i in (0..2).map(Id::from) {
 					assert_eq!(duty_roster.validator_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 					assert_eq!(duty_roster.guarantor_duty.iter().filter(|&&j| j == Chain::Parachain(i)).count(), 3);
 				}

substrate/polkadot/runtime/src/runtime/session.rs

@@ -35,7 +35,7 @@ impl StorageVec for ValidatorStorageVec {
 	const PREFIX: &'static[u8] = b"ses:val:";
 }
 
-/// Get the current set of authorities. These are the session keys.
+/// Get the current set of validators.
 pub fn validators() -> Vec<AccountId> {
 	ValidatorStorageVec::items()
 }

substrate/polkadot/runtime/src/support/storage.rs

@@ -18,14 +18,33 @@
 
 use rstd::prelude::*;
 use runtime_io::{self, twox_128};
-use codec::{Slicable, KeyedVec};
+use codec::{Input, Slicable, KeyedVec};
 
 // TODO: consider using blake256 to avoid possible preimage attack.
 
+struct IncrementalInput<'a> {
+	key: &'a [u8],
+	pos: usize,
+}
+
+impl<'a> Input for IncrementalInput<'a> {
+	fn read(&mut self, into: &mut [u8]) -> usize {
+		let len = runtime_io::read_storage(self.key, into, self.pos);
+		let read = ::rstd::cmp::min(len, into.len());
+		self.pos += read;
+		read
+	}
+}
+
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-	let raw = runtime_io::storage(&twox_128(key)[..]);
-	Slicable::from_slice(&mut &raw[..])
+	let key = twox_128(key);
+	let mut input = IncrementalInput {
+		key: &key[..],
+		pos: 0,
+	};
+
+	Slicable::decode(&mut input)
 }
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
@@ -142,12 +161,16 @@ pub trait StorageVec {
 }
 
 pub mod unhashed {
-	use super::{runtime_io, Slicable, KeyedVec, Vec};
+	use super::{runtime_io, Slicable, KeyedVec, Vec, IncrementalInput};
 
 	/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 	pub fn get<T: Slicable + Sized>(key: &[u8]) -> Option<T> {
-		let raw = runtime_io::storage(key);
-		T::from_slice(&mut &raw[..])
+		let mut input = IncrementalInput {
+			key,
+			pos: 0,
+		};
+
+		T::decode(&mut input)
 	}
 
 	/// Return the value of the item in storage under `key`, or the type's default if there is no