Minimal parachain framework part 1 (#113)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* grumbles

* update WASM checkins

* remove dependency on tokio-timer

polkadot/api/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 error-chain = "0.11"
-log = "0.3"
 polkadot-executor = { path = "../executor" }
 polkadot-runtime = { path = "../runtime" }
 polkadot-primitives = { path = "../primitives" }

polkadot/api/src/lib.rs

@@ -40,8 +40,8 @@ use polkadot_executor::Executor as LocalDispatch;
 use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
 use state_machine::OverlayedChanges;
 use primitives::{AccountId, BlockId, Hash, Index, SessionKey, Timestamp};
-use primitives::parachain::DutyRoster;
-use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall};
+use primitives::parachain::{DutyRoster, CandidateReceipt, Id as ParaId};
+use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall, ParachainsCall};
 
 error_chain! {
 	errors {
@@ -135,12 +135,21 @@ pub trait PolkadotApi {
 	/// Get the index of an account at a block.
 	fn index(&self, at: &Self::CheckedBlockId, account: AccountId) -> Result<Index>;
 
+	/// Get the active parachains at a block.
+	fn active_parachains(&self, at: &Self::CheckedBlockId) -> Result<Vec<ParaId>>;
 
-	/// Evaluate a block and see if it gives an error.
-	fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<()>;
+	/// Get the validation code of a parachain at a block. If the parachain is active, this will always return `Some`.
+	fn parachain_code(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
+
+	/// Get the chain head of a parachain. If the parachain is active, this will always return `Some`.
+	fn parachain_head(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
+
+	/// Evaluate a block. Returns true if the block is good, false if it is known to be bad,
+	/// and an error if we can't evaluate for some reason.
+	fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<bool>;
 
 	/// Create a block builder on top of the parent block.
-	fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp) -> Result<Self::BlockBuilder>;
+	fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder>;
 }
 
 /// A checked block ID used for the substrate-client implementation of CheckedBlockId;
@@ -213,15 +222,36 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
 		with_runtime!(self, at, ::runtime::Timestamp::now)
 	}
 
-	fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<()> {
-		with_runtime!(self, at, || ::runtime::Executive::execute_block(block))
+	fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<bool> {
+		use substrate_executor::error::ErrorKind as ExecErrorKind;
+
+		let res = with_runtime!(self, at, || ::runtime::Executive::execute_block(block));
+		match res {
+			Ok(()) => Ok(true),
+			Err(err) => match err.kind() {
+				&ErrorKind::Executor(ExecErrorKind::Runtime) => Ok(false),
+				_ => Err(err)
+			}
+		}
 	}
 
 	fn index(&self, at: &CheckedId, account: AccountId) -> Result<Index> {
 		with_runtime!(self, at, || ::runtime::System::account_index(account))
 	}
 
-	fn build_block(&self, parent: &CheckedId, timestamp: Timestamp) -> Result<Self::BlockBuilder> {
+	fn active_parachains(&self, at: &CheckedId) -> Result<Vec<ParaId>> {
+		with_runtime!(self, at, ::runtime::Parachains::active_parachains)
+	}
+
+	fn parachain_code(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
+		with_runtime!(self, at, || ::runtime::Parachains::parachain_code(parachain))
+	}
+
+	fn parachain_head(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
+		with_runtime!(self, at, || ::runtime::Parachains::parachain_head(parachain))
+	}
+
+	fn build_block(&self, parent: &CheckedId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder> {
 		let parent = parent.block_id();
 		let header = Header {
 			parent_hash: self.block_hash_from_id(parent)?.ok_or(ErrorKind::UnknownBlock(*parent))?,
@@ -239,6 +269,14 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
 					function: Call::Timestamp(TimestampCall::set(timestamp)),
 				},
 				signature: Default::default(),
+			},
+			UncheckedExtrinsic {
+				extrinsic: Extrinsic {
+					signed: Default::default(),
+					index: Default::default(),
+					function: Call::Parachains(ParachainsCall::set_heads(parachains)),
+				},
+				signature: Default::default(),
 			}
 		];
 
@@ -275,7 +313,7 @@ pub struct ClientBlockBuilder<S> {
 impl<S: state_machine::Backend> ClientBlockBuilder<S>
 	where S::Error: Into<client::error::Error>
 {
-	// initialises a block ready to allow extrinsics to be applied.
+	// initialises a block, ready to allow extrinsics to be applied.
 	fn initialise_block(&mut self) -> Result<()> {
 		let result = {
 			let mut ext = state_machine::Ext::new(&mut self.changes, &self.state);
@@ -406,7 +444,7 @@ mod tests {
 		let client = client();
 
 		let id = client.check_id(BlockId::Number(0)).unwrap();
-		let block_builder = client.build_block(&id, 1_000_000).unwrap();
+		let block_builder = client.build_block(&id, 1_000_000, Vec::new()).unwrap();
 		let block = block_builder.bake();
 
 		assert_eq!(block.header.number, 1);

polkadot/collator/Cargo.toml

@@ -1,10 +1,12 @@
 [package]
 name = "polkadot-collator"
 version = "0.1.0"
-authors = ["Parity Technologies <rphmeier@gmail.com>"]
+authors = ["Parity Technologies <admin@parity.io>"]
 description = "Abstract collation logic"
 
 [dependencies]
 futures = "0.1.17"
+substrate-codec = { path = "../../substrate/codec", version = "0.1" }
 substrate-primitives = { path = "../../substrate/primitives", version = "0.1" }
 polkadot-primitives = { path = "../primitives", version = "0.1" }
+polkadot-parachain = { path = "../parachain", version = "0.1" }

polkadot/collator/src/lib.rs

@@ -45,6 +45,7 @@
 //! to be performed, as the collation logic itself.
 
 extern crate futures;
+extern crate substrate_codec as codec;
 extern crate substrate_primitives as primitives;
 extern crate polkadot_primitives;
 
@@ -82,7 +83,6 @@ pub trait RelayChainContext {
 }
 
 /// Collate the necessary ingress queue using the given context.
-// TODO: impl trait
 pub fn collate_ingress<'a, R>(relay_context: R)
 	-> Box<Future<Item=ConsolidatedIngress, Error=R::Error> + 'a>
 	where
@@ -105,7 +105,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
 	// and then by the parachain ID.
 	//
 	// then transform that into the consolidated egress queue.
-	let future = stream::futures_unordered(egress_fetch)
+	Box::new(stream::futures_unordered(egress_fetch)
 		.fold(BTreeMap::new(), |mut map, (routing_id, egresses)| {
 			for (depth, egress) in egresses.into_iter().rev().enumerate() {
 				let depth = -(depth as i64);
@@ -116,9 +116,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
 		})
 		.map(|ordered| ordered.into_iter().map(|((_, id), egress)| (id, egress)))
 		.map(|i| i.collect::<Vec<_>>())
-		.map(ConsolidatedIngress);
-
-	Box::new(future)
+		.map(ConsolidatedIngress))
 }
 
 /// Produce a candidate for the parachain.

polkadot/consensus/Cargo.toml

@@ -13,6 +13,7 @@ log = "0.3"
 exit-future = "0.1"
 polkadot-api = { path = "../api" }
 polkadot-collator = { path = "../collator" }
+polkadot-parachain = { path = "../parachain" }
 polkadot-primitives = { path = "../primitives" }
 polkadot-runtime = { path = "../runtime" }
 polkadot-statement-table = { path = "../statement-table" }

polkadot/consensus/src/collation.rs

@@ -0,0 +1,176 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Validator-side view of collation.
+//!
+//! This module contains type definitions, a trait for a batch of collators, and a trait for
+//! attempting to fetch a collation repeatedly until a valid one is obtained.
+
+use std::sync::Arc;
+
+use polkadot_api::PolkadotApi;
+use polkadot_primitives::{Hash, AccountId};
+use polkadot_primitives::parachain::{Id as ParaId, Chain, BlockData, Extrinsic, CandidateReceipt};
+
+use futures::prelude::*;
+
+/// A full collation.
+pub struct Collation {
+	/// Block data.
+	pub block_data: BlockData,
+	/// The candidate receipt itself.
+	pub receipt: CandidateReceipt,
+}
+
+/// Encapsulates connections to collators and allows collation on any parachain.
+///
+/// This is expected to be a lightweight, shared type like an `Arc`.
+pub trait Collators: Clone {
+	/// Errors when producing collations.
+	type Error;
+	/// A full collation.
+	type Collation: IntoFuture<Item=Collation,Error=Self::Error>;
+
+	/// Collate on a specific parachain, building on a given relay chain parent hash.
+	fn collate(&self, parachain: ParaId, relay_parent: Hash) -> Self::Collation;
+
+	/// Note a bad collator. TODO: take proof
+	fn note_bad_collator(&self, collator: AccountId);
+}
+
+/// A future which resolves when a collation is available.
+///
+/// This future is fused.
+pub struct CollationFetch<C: Collators, P: PolkadotApi> {
+	parachain: Option<ParaId>,
+	relay_parent_hash: Hash,
+	relay_parent: P::CheckedBlockId,
+	collators: C,
+	live_fetch: Option<<C::Collation as IntoFuture>::Future>,
+	client: Arc<P>,
+}
+
+impl<C: Collators, P: PolkadotApi> CollationFetch<C, P> {
+	/// Create a new collation fetcher for the given chain.
+	pub fn new(parachain: Chain, relay_parent: P::CheckedBlockId, relay_parent_hash: Hash, collators: C, client: Arc<P>) -> Self {
+		CollationFetch {
+			relay_parent_hash,
+			relay_parent,
+			collators,
+			client,
+			parachain: match parachain {
+				Chain::Parachain(id) => Some(id),
+				Chain::Relay => None,
+			},
+			live_fetch: None,
+		}
+	}
+}
+
+impl<C: Collators, P: PolkadotApi> Future for CollationFetch<C, P> {
+	type Item = (Collation, Extrinsic);
+	type Error = C::Error;
+
+	fn poll(&mut self) -> Poll<(Collation, Extrinsic), C::Error> {
+		let parachain = match self.parachain.as_ref() {
+			Some(p) => p.clone(),
+			None => return Ok(Async::NotReady),
+		};
+
+		loop {
+			let x = {
+				let (r, c)  = (self.relay_parent_hash, &self.collators);
+				let poll = self.live_fetch
+					.get_or_insert_with(move || c.collate(parachain, r).into_future())
+					.poll();
+
+				if let Err(_) = poll { self.parachain = None }
+				try_ready!(poll)
+			};
+
+			match validate_collation(&*self.client, &self.relay_parent, &x) {
+				Ok(()) => {
+					self.parachain = None;
+
+					// TODO: generate extrinsic while verifying.
+					return Ok(Async::Ready((x, Extrinsic)));
+				}
+				Err(e) => {
+					debug!("Failed to validate parachain due to API error: {}", e);
+
+					// just continue if we got a bad collation or failed to validate
+					self.live_fetch = None;
+					self.collators.note_bad_collator(x.receipt.collator)
+				}
+			}
+		}
+	}
+}
+
+// Errors that can occur when validating a parachain.
+error_chain! {
+	types { Error, ErrorKind, ResultExt; }
+
+	errors {
+		InactiveParachain(id: ParaId) {
+			description("Collated for inactive parachain"),
+			display("Collated for inactive parachain: {:?}", id),
+		}
+		ValidationFailure {
+			description("Parachain candidate failed validation."),
+			display("Parachain candidate failed validation."),
+		}
+		WrongHeadData(expected: Vec<u8>, got: Vec<u8>) {
+			description("Parachain validation produced wrong head data."),
+			display("Parachain validation produced wrong head data (expected: {:?}, got {:?}", expected, got),
+		}
+	}
+
+	links {
+		PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
+	}
+}
+
+/// Check whether a given collation is valid. Returns `Ok`  on success, error otherwise.
+pub fn validate_collation<P: PolkadotApi>(client: &P, relay_parent: &P::CheckedBlockId, collation: &Collation) -> Result<(), Error> {
+	use parachain::{self, ValidationParams};
+
+	let para_id = collation.receipt.parachain_index;
+	let validation_code = client.parachain_code(relay_parent, para_id)?
+		.ok_or_else(|| ErrorKind::InactiveParachain(para_id))?;
+
+	let chain_head = client.parachain_head(relay_parent, para_id)?
+		.ok_or_else(|| ErrorKind::InactiveParachain(para_id))?;
+
+	let params = ValidationParams {
+		parent_head: chain_head,
+		block_data: collation.block_data.0.clone(),
+	};
+
+	match parachain::wasm::validate_candidate(&validation_code, params) {
+		Ok(result) => {
+			if result.head_data == collation.receipt.head_data.0 {
+				Ok(())
+			} else {
+				Err(ErrorKind::WrongHeadData(
+					collation.receipt.head_data.0.clone(),
+					result.head_data
+				).into())
+			}
+		}
+		Err(_) => Err(ErrorKind::ValidationFailure.into())
+	}
+}

polkadot/consensus/src/dynamic_inclusion.rs

@@ -0,0 +1,130 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Dynamic inclusion threshold over time.
+
+use std::time::{Duration, Instant};
+
+fn duration_to_micros(duration: &Duration) -> u64 {
+	duration.as_secs() * 1_000_000 + (duration.subsec_nanos() / 1000) as u64
+}
+
+/// Dynamic inclusion threshold over time.
+///
+/// The acceptable proportion of parachains which must have parachain candidates
+/// reduces over time (eventually going to zero).
+#[derive(Debug, Clone)]
+pub struct DynamicInclusion {
+	start: Instant,
+	y: u64,
+	m: u64,
+}
+
+impl DynamicInclusion {
+	/// Constructs a new dynamic inclusion threshold calculator based on the time now,
+	/// how many parachain candidates are required at the beginning, and when an empty
+	/// block will be allowed.
+	pub fn new(initial: usize, start: Instant, allow_empty: Duration) -> Self {
+		// linear function f(n_candidates) -> valid after microseconds
+		// f(0) = allow_empty
+		// f(initial) = 0
+		// m is actually the negative slope to avoid using signed arithmetic.
+		let (y, m) = if initial != 0 {
+			let y = duration_to_micros(&allow_empty);
+
+			(y, y / initial as u64)
+		} else {
+			(0, 0)
+		};
+
+		DynamicInclusion {
+			start,
+			y,
+			m,
+		}
+	}
+
+	/// Returns the duration from `now` after which the amount of included parachain candidates
+	/// would be enough, or `None` if it is sufficient now.
+	///
+	/// Panics if `now` is earlier than the `start`.
+	pub fn acceptable_in(&self, now: Instant, included: usize) -> Option<Duration> {
+		let elapsed = now.duration_since(self.start);
+		let elapsed = duration_to_micros(&elapsed);
+
+		let valid_after = self.y.saturating_sub(self.m * included as u64);
+
+		if elapsed >= valid_after {
+			None
+		} else {
+			Some(Duration::from_millis((valid_after - elapsed) as u64 / 1000))
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn full_immediately_allowed() {
+		let now = Instant::now();
+
+		let dynamic = DynamicInclusion::new(
+			10,
+			now,
+			Duration::from_millis(4000),
+		);
+
+		assert!(dynamic.acceptable_in(now, 10).is_none());
+		assert!(dynamic.acceptable_in(now, 11).is_none());
+		assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 10).is_none());
+	}
+
+	#[test]
+	fn half_allowed_halfway() {
+		let now = Instant::now();
+
+		let dynamic = DynamicInclusion::new(
+			10,
+			now,
+			Duration::from_millis(4000),
+		);
+
+		assert_eq!(dynamic.acceptable_in(now, 5), Some(Duration::from_millis(2000)));
+		assert!(dynamic.acceptable_in(now + Duration::from_millis(2000), 5).is_none());
+		assert!(dynamic.acceptable_in(now + Duration::from_millis(3000), 5).is_none());
+		assert!(dynamic.acceptable_in(now + Duration::from_millis(4000), 5).is_none());
+	}
+
+	#[test]
+	fn zero_initial_is_flat() {
+		let now = Instant::now();
+
+		let dynamic = DynamicInclusion::new(
+			0,
+			now,
+			Duration::from_secs(10_000),
+		);
+
+		for i in 0..10_001 {
+			let now = now + Duration::from_secs(i);
+			assert!(dynamic.acceptable_in(now, 0).is_none());
+			assert!(dynamic.acceptable_in(now, 1).is_none());
+			assert!(dynamic.acceptable_in(now, 10).is_none());
+		}
+	}
+}

polkadot/consensus/src/error.rs

@@ -16,45 +16,30 @@
 
 //! Errors that can occur during the consensus process.
 
-use primitives::block::{HeaderHash, Number};
+use polkadot_primitives::AccountId;
+
 error_chain! {
 	links {
 		PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
 		Bft(::bft::Error, ::bft::ErrorKind);
 	}
 
-	foreign_links {
-		Io(::std::io::Error);
-		SharedIo(::futures::future::SharedError<::std::io::Error>);
-	}
-
 	errors {
 		InvalidDutyRosterLength(expected: usize, got: usize) {
 			description("Duty Roster had invalid length"),
 			display("Invalid duty roster length: expected {}, got {}", expected, got),
 		}
-		ProposalNotForPolkadot {
-			description("Proposal provided not a Polkadot block."),
-			display("Proposal provided not a Polkadot block."),
+		NotValidator(id: AccountId) {
+			description("Local account ID not a validator at this block."),
+			display("Local account ID ({:?}) not a validator at this block.", id),
 		}
-		TimestampInFuture {
-			description("Proposal had timestamp too far in the future."),
-			display("Proposal had timestamp too far in the future."),
+		PrematureDestruction {
+			description("Proposer destroyed before finishing proposing or evaluating"),
+			display("Proposer destroyed before finishing proposing or evaluating"),
 		}
-		WrongParentHash(expected: HeaderHash, got: HeaderHash) {
-			description("Proposal had wrong parent hash."),
-			display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got),
-		}
-		WrongNumber(expected: Number, got: Number) {
-			description("Proposal had wrong number."),
-			display("Proposal had wrong number. Expected {:?}, got {:?}", expected, got),
-		}
-		ProposalTooLarge(size: usize) {
-			description("Proposal exceeded the maximum size."),
-			display(
-				"Proposal exceeded the maximum size of {} by {} bytes.",
-				::MAX_TRANSACTIONS_SIZE, ::MAX_TRANSACTIONS_SIZE.saturating_sub(*size)
-			),
+		Timer(e: String) {
+			description("Failed to register or resolve async timer."),
+			display("Timer failed: {}", e),
 		}
 		Executor(e: ::futures::future::ExecuteErrorKind) {
 			description("Unable to dispatch agreement future"),

polkadot/consensus/src/evaluation.rs

@@ -0,0 +1,135 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot block evaluation and evaluation errors.
+
+use super::MAX_TRANSACTIONS_SIZE;
+
+use codec::Slicable;
+use polkadot_runtime::Block as PolkadotGenericBlock;
+use polkadot_primitives::Timestamp;
+use polkadot_primitives::parachain::Id as ParaId;
+use primitives::block::{Block as SubstrateBlock, HeaderHash, Number as BlockNumber};
+use transaction_pool::PolkadotBlock;
+
+error_chain! {
+	links {
+		PolkadotApi(::polkadot_api::Error, ::polkadot_api::ErrorKind);
+	}
+
+	errors {
+		ProposalNotForPolkadot {
+			description("Proposal provided not a Polkadot block."),
+			display("Proposal provided not a Polkadot block."),
+		}
+		TimestampInFuture {
+			description("Proposal had timestamp too far in the future."),
+			display("Proposal had timestamp too far in the future."),
+		}
+		TooManyCandidates(expected: usize, got: usize) {
+			description("Proposal included more candidates than is possible."),
+			display("Proposal included {} candidates for {} parachains", got, expected),
+		}
+		ParachainOutOfOrder {
+			description("Proposal included parachains out of order."),
+			display("Proposal included parachains out of order."),
+		}
+		UnknownParachain(id: ParaId) {
+			description("Proposal included unregistered parachain."),
+			display("Proposal included unregistered parachain {:?}", id),
+		}
+		WrongParentHash(expected: HeaderHash, got: HeaderHash) {
+			description("Proposal had wrong parent hash."),
+			display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got),
+		}
+		WrongNumber(expected: BlockNumber, got: BlockNumber) {
+			description("Proposal had wrong number."),
+			display("Proposal had wrong number. Expected {:?}, got {:?}", expected, got),
+		}
+		ProposalTooLarge(size: usize) {
+			description("Proposal exceeded the maximum size."),
+			display(
+				"Proposal exceeded the maximum size of {} by {} bytes.",
+				MAX_TRANSACTIONS_SIZE, MAX_TRANSACTIONS_SIZE.saturating_sub(*size)
+			),
+		}
+	}
+}
+
+/// Attempt to evaluate a substrate block as a polkadot block, returning error
+/// upon any initial validity checks failing.
+pub fn evaluate_initial(
+	proposal: &SubstrateBlock,
+	now: Timestamp,
+	parent_hash: &HeaderHash,
+	parent_number: BlockNumber,
+	active_parachains: &[ParaId],
+) -> Result<PolkadotBlock> {
+	const MAX_TIMESTAMP_DRIFT: Timestamp = 60;
+
+	let encoded = Slicable::encode(proposal);
+	let proposal = PolkadotGenericBlock::decode(&mut &encoded[..])
+		.and_then(|b| PolkadotBlock::from(b).ok())
+		.ok_or_else(|| ErrorKind::ProposalNotForPolkadot)?;
+
+	let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| {
+		a + Slicable::encode(tx).len()
+	});
+
+	if transactions_size > MAX_TRANSACTIONS_SIZE {
+		bail!(ErrorKind::ProposalTooLarge(transactions_size))
+	}
+
+	if proposal.header.parent_hash != *parent_hash {
+		bail!(ErrorKind::WrongParentHash(*parent_hash, proposal.header.parent_hash));
+	}
+
+	if proposal.header.number != parent_number + 1 {
+		bail!(ErrorKind::WrongNumber(parent_number + 1, proposal.header.number));
+	}
+
+	let block_timestamp = proposal.timestamp();
+
+	// lenient maximum -- small drifts will just be delayed using a timer.
+	if block_timestamp > now + MAX_TIMESTAMP_DRIFT {
+		bail!(ErrorKind::TimestampInFuture)
+	}
+
+	{
+		let n_parachains = active_parachains.len();
+		if proposal.parachain_heads().len() > n_parachains {
+			bail!(ErrorKind::TooManyCandidates(n_parachains, proposal.parachain_heads().len()));
+		}
+
+		let mut last_id = None;
+		let mut iter = active_parachains.iter();
+		for head in proposal.parachain_heads() {
+			// proposed heads must be ascending order by parachain ID without duplicate.
+			if last_id.as_ref().map_or(false, |x| x >= &head.parachain_index) {
+				bail!(ErrorKind::ParachainOutOfOrder);
+			}
+
+			if !iter.any(|x| x == &head.parachain_index) {
+				// must be unknown since active parachains are always sorted.
+				bail!(ErrorKind::UnknownParachain(head.parachain_index))
+			}
+
+			last_id = Some(head.parachain_index);
+		}
+	}
+
+	Ok(proposal)
+}

polkadot/consensus/src/lib.rs

@@ -29,15 +29,16 @@
 //!
 //! Groups themselves may be compromised by malicious authorities.
 
-extern crate futures;
 extern crate ed25519;
 extern crate parking_lot;
 extern crate polkadot_api;
 extern crate polkadot_collator as collator;
 extern crate polkadot_statement_table as table;
+extern crate polkadot_parachain as parachain;
 extern crate polkadot_primitives;
 extern crate polkadot_transaction_pool as transaction_pool;
 extern crate polkadot_runtime;
+
 extern crate substrate_bft as bft;
 extern crate substrate_codec as codec;
 extern crate substrate_primitives as primitives;
@@ -46,46 +47,60 @@ extern crate substrate_network;
 
 extern crate exit_future;
 extern crate tokio_core;
-extern crate substrate_keyring;
 extern crate substrate_client as client;
 
 #[macro_use]
 extern crate error_chain;
+
+#[macro_use]
+extern crate futures;
+
 #[macro_use]
 extern crate log;
 
+#[cfg(test)]
+extern crate substrate_keyring;
+
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
+use std::time::{Duration, Instant};
 
 use codec::Slicable;
-use table::{Table, Context as TableContextTrait};
 use table::generic::Statement as GenericStatement;
 use runtime_support::Hashable;
 use polkadot_api::{PolkadotApi, BlockBuilder};
 use polkadot_primitives::{Hash, Timestamp};
-use polkadot_primitives::parachain::{Id as ParaId, DutyRoster, BlockData, Extrinsic, CandidateReceipt};
-use polkadot_runtime::Block as PolkadotGenericBlock;
+use polkadot_primitives::parachain::{Id as ParaId, Chain, DutyRoster, BlockData, Extrinsic, CandidateReceipt};
 use primitives::block::{Block as SubstrateBlock, Header as SubstrateHeader, HeaderHash, Id as BlockId, Number as BlockNumber};
 use primitives::AuthorityId;
-use transaction_pool::{Ready, TransactionPool, PolkadotBlock};
+use transaction_pool::{Ready, TransactionPool};
+use tokio_core::reactor::{Handle, Timeout, Interval};
 
 use futures::prelude::*;
-use futures::future;
-use future::Shared;
+use futures::future::{self, Shared};
 use parking_lot::Mutex;
-use tokio_core::reactor::{Handle, Timeout};
+use collation::CollationFetch;
+use dynamic_inclusion::DynamicInclusion;
 
+pub use self::collation::{Collators, Collation};
 pub use self::error::{ErrorKind, Error};
+pub use self::shared_table::{SharedTable, StatementSource, StatementProducer, ProducedStatements};
 pub use service::Service;
 
+mod collation;
+mod dynamic_inclusion;
+mod evaluation;
 mod error;
 mod service;
+mod shared_table;
 
 // block size limit.
 const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
 
 /// A handle to a statement table router.
-pub trait TableRouter {
+///
+/// This is expected to be a lightweight, shared type like an `Arc`.
+pub trait TableRouter: Clone {
 	/// Errors when fetching data from the network.
 	type Error;
 	/// Future that resolves when candidate data is fetched.
@@ -93,7 +108,7 @@ pub trait TableRouter {
 	/// Future that resolves when extrinsic candidate data is fetched.
 	type FetchExtrinsic: IntoFuture<Item=Extrinsic,Error=Self::Error>;
 
-	/// Note local candidate data.
+	/// Note local candidate data, making it available on the network to other validators.
 	fn local_candidate_data(&self, hash: Hash, block_data: BlockData, extrinsic: Extrinsic);
 
 	/// Fetch block data for a specific candidate.
@@ -126,46 +141,6 @@ pub struct GroupInfo {
 	pub needed_availability: usize,
 }
 
-struct TableContext {
-	parent_hash: Hash,
-	key: Arc<ed25519::Pair>,
-	groups: HashMap<ParaId, GroupInfo>,
-}
-
-impl table::Context for TableContext {
-	fn is_member_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
-		self.groups.get(group).map_or(false, |g| g.validity_guarantors.contains(authority))
-	}
-
-	fn is_availability_guarantor_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
-		self.groups.get(group).map_or(false, |g| g.availability_guarantors.contains(authority))
-	}
-
-	fn requisite_votes(&self, group: &ParaId) -> (usize, usize) {
-		self.groups.get(group).map_or(
-			(usize::max_value(), usize::max_value()),
-			|g| (g.needed_validity, g.needed_availability),
-		)
-	}
-}
-
-impl TableContext {
-	fn local_id(&self) -> AuthorityId {
-		self.key.public().0
-	}
-
-	fn sign_statement(&self, statement: table::Statement) -> table::SignedStatement {
-		let signature = sign_table_statement(&statement, &self.key, &self.parent_hash).into();
-		let local_id = self.key.public().0;
-
-		table::SignedStatement {
-			statement,
-			signature,
-			sender: local_id,
-		}
-	}
-}
-
 /// Sign a table statement against a parent hash.
 /// The actual message signed is the encoded statement concatenated with the
 /// parent hash.
@@ -185,247 +160,7 @@ pub fn sign_table_statement(statement: &table::Statement, key: &ed25519::Pair, p
 	key.sign(&encoded)
 }
 
-// A shared table object.
-struct SharedTableInner {
-	table: Table<TableContext>,
-	proposed_digest: Option<Hash>,
-	checked_validity: HashSet<Hash>,
-	checked_availability: HashSet<Hash>,
-}
-
-impl SharedTableInner {
-	// Import a single statement. Provide a handle to a table router.
-	fn import_statement<R: TableRouter>(
-		&mut self,
-		context: &TableContext,
-		router: &R,
-		statement: table::SignedStatement,
-		received_from: Option<AuthorityId>,
-	) -> StatementProducer<<R::FetchCandidate as IntoFuture>::Future, <R::FetchExtrinsic as IntoFuture>::Future> {
-		let mut producer = StatementProducer {
-			fetch_block_data: None,
-			fetch_extrinsic: None,
-			produced_statements: Default::default(),
-			_key: context.key.clone(),
-		};
-
-		let summary = match self.table.import_statement(context, statement, received_from) {
-			Some(summary) => summary,
-			None => return producer,
-		};
-
-		let local_id = context.local_id();
-		let is_validity_member = context.is_member_of(&local_id, &summary.group_id);
-		let is_availability_member =
-			context.is_availability_guarantor_of(&local_id, &summary.group_id);
-
-		let digest = &summary.candidate;
-
-		// TODO: consider a strategy based on the number of candidate votes as well.
-		// only check validity if this wasn't locally proposed.
-		let checking_validity = is_validity_member
-			&& self.proposed_digest.as_ref().map_or(true, |d| d != digest)
-			&& self.checked_validity.insert(digest.clone());
-
-		let checking_availability = is_availability_member && self.checked_availability.insert(digest.clone());
-
-		if checking_validity || checking_availability {
-			match self.table.get_candidate(&digest) {
-				None => {} // TODO: handle table inconsistency somehow?
-				Some(candidate) => {
-					if checking_validity {
-						producer.fetch_block_data = Some(router.fetch_block_data(candidate).into_future().fuse());
-					}
-
-					if checking_availability {
-						producer.fetch_extrinsic = Some(router.fetch_extrinsic_data(candidate).into_future().fuse());
-					}
-				}
-			}
-		}
-
-		producer
-	}
-}
-
-/// Produced statements about a specific candidate.
-/// Both may be `None`.
-#[derive(Default)]
-pub struct ProducedStatements {
-	/// A statement about the validity of the candidate.
-	pub validity: Option<table::Statement>,
-	/// A statement about the availability of the candidate.
-	pub availability: Option<table::Statement>,
-}
-
-/// Future that produces statements about a specific candidate.
-pub struct StatementProducer<D: Future, E: Future> {
-	fetch_block_data: Option<future::Fuse<D>>,
-	fetch_extrinsic: Option<future::Fuse<E>>,
-	produced_statements: ProducedStatements,
-	_key: Arc<ed25519::Pair>,
-}
-
-impl<D, E, Err> Future for StatementProducer<D, E>
-	where
-		D: Future<Item=BlockData,Error=Err>,
-		E: Future<Item=Extrinsic,Error=Err>,
-{
-	type Item = ProducedStatements;
-	type Error = Err;
-
-	fn poll(&mut self) -> Poll<ProducedStatements, Err> {
-		let mut done = true;
-		if let Some(ref mut fetch_block_data) = self.fetch_block_data {
-			match fetch_block_data.poll()? {
-				Async::Ready(_block_data) => {
-					// TODO [PoC-2] : validate block data here and make statement.
-				},
-				Async::NotReady => {
-					done = false;
-				}
-			}
-		}
-
-		if let Some(ref mut fetch_extrinsic) = self.fetch_extrinsic {
-			match fetch_extrinsic.poll()? {
-				Async::Ready(_extrinsic) => {
-					// TODO [PoC-2]: guarantee availability of data and make statment.
-				}
-				Async::NotReady => {
-					done = false;
-				}
-			}
-		}
-
-		if done {
-			Ok(Async::Ready(::std::mem::replace(&mut self.produced_statements, Default::default())))
-		} else {
-			Ok(Async::NotReady)
-		}
-	}
-}
-
-/// A shared table object.
-pub struct SharedTable {
-	context: Arc<TableContext>,
-	inner: Arc<Mutex<SharedTableInner>>,
-}
-
-impl Clone for SharedTable {
-	fn clone(&self) -> Self {
-		SharedTable {
-			context: self.context.clone(),
-			inner: self.inner.clone(),
-		}
-	}
-}
-
-impl SharedTable {
-	/// Create a new shared table.
-	///
-	/// Provide the key to sign with, and the parent hash of the relay chain
-	/// block being built.
-	pub fn new(groups: HashMap<ParaId, GroupInfo>, key: Arc<ed25519::Pair>, parent_hash: Hash) -> Self {
-		SharedTable {
-			context: Arc::new(TableContext { groups, key, parent_hash }),
-			inner: Arc::new(Mutex::new(SharedTableInner {
-				table: Table::default(),
-				proposed_digest: None,
-				checked_validity: HashSet::new(),
-				checked_availability: HashSet::new(),
-			}))
-		}
-	}
-
-	/// Get group info.
-	pub fn group_info(&self) -> &HashMap<ParaId, GroupInfo> {
-		&self.context.groups
-	}
-
-	/// Import a single statement. Provide a handle to a table router
-	/// for dispatching any other requests which come up.
-	pub fn import_statement<R: TableRouter>(
-		&self,
-		router: &R,
-		statement: table::SignedStatement,
-		received_from: Option<AuthorityId>,
-	) -> StatementProducer<<R::FetchCandidate as IntoFuture>::Future, <R::FetchExtrinsic as IntoFuture>::Future> {
-		self.inner.lock().import_statement(&*self.context, router, statement, received_from)
-	}
-
-	/// Sign and import a local statement.
-	pub fn sign_and_import<R: TableRouter>(
-		&self,
-		router: &R,
-		statement: table::Statement,
-	) -> StatementProducer<<R::FetchCandidate as IntoFuture>::Future, <R::FetchExtrinsic as IntoFuture>::Future> {
-		let proposed_digest = match statement {
-			GenericStatement::Candidate(ref c) => Some(c.hash()),
-			_ => None,
-		};
-
-		let signed_statement = self.context.sign_statement(statement);
-
-		let mut inner = self.inner.lock();
-		if proposed_digest.is_some() {
-			inner.proposed_digest = proposed_digest;
-		}
-
-		inner.import_statement(&*self.context, router, signed_statement, None)
-	}
-
-	/// Import many statements at once.
-	///
-	/// Provide an iterator yielding pairs of (statement, received_from).
-	pub fn import_statements<R, I, U>(&self, router: &R, iterable: I) -> U
-		where
-			R: TableRouter,
-			I: IntoIterator<Item=(table::SignedStatement, Option<AuthorityId>)>,
-			U: ::std::iter::FromIterator<StatementProducer<
-				<R::FetchCandidate as IntoFuture>::Future,
-				<R::FetchExtrinsic as IntoFuture>::Future>
-			>,
-	{
-		let mut inner = self.inner.lock();
-
-		iterable.into_iter().map(move |(statement, received_from)| {
-			inner.import_statement(&*self.context, router, statement, received_from)
-		}).collect()
-	}
-
-	/// Check if a proposal is valid.
-	pub fn proposal_valid(&self, _proposal: &SubstrateBlock) -> bool {
-		false // TODO
-	}
-
-	/// Execute a closure using a specific candidate.
-	///
-	/// Deadlocks if called recursively.
-	pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
-		where F: FnOnce(Option<&CandidateReceipt>) -> U
-	{
-		let inner = self.inner.lock();
-		f(inner.table.get_candidate(digest))
-	}
-
-	/// Get all witnessed misbehavior.
-	pub fn get_misbehavior(&self) -> HashMap<AuthorityId, table::Misbehavior> {
-		self.inner.lock().table.get_misbehavior().clone()
-	}
-
-	/// Fill a statement batch.
-	pub fn fill_batch<B: table::StatementBatch>(&self, batch: &mut B) {
-		self.inner.lock().table.fill_batch(batch);
-	}
-
-	/// Get the local proposed block's hash.
-	pub fn proposed_hash(&self) -> Option<Hash> {
-		self.inner.lock().proposed_digest.clone()
-	}
-}
-
-fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId]) -> Result<HashMap<ParaId, GroupInfo>, Error> {
+fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId], local_id: AuthorityId) -> Result<(HashMap<ParaId, GroupInfo>, LocalDuty), Error> {
 	if roster.validator_duty.len() != authorities.len() {
 		bail!(ErrorKind::InvalidDutyRosterLength(authorities.len(), roster.validator_duty.len()))
 	}
@@ -434,11 +169,14 @@ fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId]) -> Result<Ha
 		bail!(ErrorKind::InvalidDutyRosterLength(authorities.len(), roster.guarantor_duty.len()))
 	}
 
+	let mut local_validation = None;
 	let mut map = HashMap::new();
 
 	let duty_iter = authorities.iter().zip(&roster.validator_duty).zip(&roster.guarantor_duty);
 	for ((authority, v_duty), a_duty) in duty_iter {
-		use polkadot_primitives::parachain::Chain;
+		if authority == &local_id {
+			local_validation = Some(v_duty.clone());
+		}
 
 		match *v_duty {
 			Chain::Relay => {}, // does nothing for now.
@@ -467,23 +205,45 @@ fn make_group_info(roster: DutyRoster, authorities: &[AuthorityId]) -> Result<Ha
 		live_group.needed_availability = availability_len / 2 + availability_len % 2;
 	}
 
-	Ok(map)
+	match local_validation {
+		Some(local_validation) => {
+			let local_duty = LocalDuty {
+				validation: local_validation,
+			};
+
+			Ok((map, local_duty))
+		}
+		None => bail!(ErrorKind::NotValidator(local_id)),
+	}
+}
+
+fn timer_error(e: &::std::io::Error) -> Error {
+	ErrorKind::Timer(format!("{}", e)).into()
 }
 
 /// Polkadot proposer factory.
-pub struct ProposerFactory<C, N> {
+pub struct ProposerFactory<C, N, P> {
 	/// The client instance.
 	pub client: Arc<C>,
 	/// The transaction pool.
 	pub transaction_pool: Arc<Mutex<TransactionPool>>,
 	/// The backing network handle.
 	pub network: N,
-	/// Handle to the underlying tokio-core.
+	/// Parachain collators.
+	pub collators: P,
+	/// The timer used to schedule proposal intervals.
 	pub handle: Handle,
+	/// The duration after which parachain-empty blocks will be allowed.
+	pub parachain_empty_duration: Duration,
 }
 
-impl<C: PolkadotApi, N: Network> bft::ProposerFactory for ProposerFactory<C, N> {
-	type Proposer = Proposer<C, N::TableRouter>;
+impl<C, N, P> bft::ProposerFactory for ProposerFactory<C, N, P>
+	where
+		C: PolkadotApi,
+		N: Network,
+		P: Collators,
+{
+	type Proposer = Proposer<C, N::TableRouter, P>;
 	type Error = Error;
 
 	fn init(&self, parent_header: &SubstrateHeader, authorities: &[AuthorityId], sign_with: Arc<ed25519::Pair>) -> Result<Self::Proposer, Error> {
@@ -497,135 +257,224 @@ impl<C: PolkadotApi, N: Network> bft::ProposerFactory for ProposerFactory<C, N>
 		let duty_roster = self.client.duty_roster(&checked_id)?;
 		let random_seed = self.client.random_seed(&checked_id)?;
 
-		let group_info = make_group_info(duty_roster, authorities)?;
+		let (group_info, local_duty) = make_group_info(
+			duty_roster,
+			authorities,
+			sign_with.public().0,
+		)?;
+
+		let active_parachains = self.client.active_parachains(&checked_id)?;
+
+		let n_parachains = active_parachains.len();
 		let table = Arc::new(SharedTable::new(group_info, sign_with.clone(), parent_hash));
 		let router = self.network.table_router(table.clone());
+		let dynamic_inclusion = DynamicInclusion::new(
+			n_parachains,
+			Instant::now(),
+			self.parachain_empty_duration.clone(),
+		);
 
-		let timeout = Timeout::new(DELAY_UNTIL, &self.handle)?;
+		let timeout = Timeout::new(DELAY_UNTIL, &self.handle)
+			.map_err(|e| timer_error(&e))?;
 
 		debug!(target: "bft", "Initialising consensus proposer. Refusing to evaluate for {:?} from now.",
 			DELAY_UNTIL);
 
 		// TODO [PoC-2]: kick off collation process.
 		Ok(Proposer {
-			parent_hash,
-			parent_number: parent_header.number,
-			parent_id: checked_id,
-			random_seed,
-			local_key: sign_with,
 			client: self.client.clone(),
-			transaction_pool: self.transaction_pool.clone(),
+			collators: self.collators.clone(),
 			delay: timeout.shared(),
-			_table: table,
-			_router: router,
+			handle: self.handle.clone(),
+			dynamic_inclusion,
+			local_duty,
+			local_key: sign_with,
+			parent_hash,
+			parent_id: checked_id,
+			parent_number: parent_header.number,
+			random_seed,
+			router,
+			table,
+			transaction_pool: self.transaction_pool.clone(),
 		})
 	}
 }
 
-fn current_timestamp() -> Timestamp {
-	use std::time;
-
-	time::SystemTime::now().duration_since(time::UNIX_EPOCH)
-		.expect("now always later than unix epoch; qed")
-		.as_secs()
+struct LocalDuty {
+	validation: Chain,
 }
 
 /// The Polkadot proposer logic.
-pub struct Proposer<C: PolkadotApi, R> {
-	parent_hash: HeaderHash,
-	parent_number: BlockNumber,
-	parent_id: C::CheckedBlockId,
-	random_seed: Hash,
+pub struct Proposer<C: PolkadotApi, R, P> {
 	client: Arc<C>,
-	local_key: Arc<ed25519::Pair>,
-	transaction_pool: Arc<Mutex<TransactionPool>>,
+	collators: P,
 	delay: Shared<Timeout>,
-	_table: Arc<SharedTable>,
-	_router: R,
+	dynamic_inclusion: DynamicInclusion,
+	handle: Handle,
+	local_duty: LocalDuty,
+	local_key: Arc<ed25519::Pair>,
+	parent_hash: HeaderHash,
+	parent_id: C::CheckedBlockId,
+	parent_number: BlockNumber,
+	random_seed: Hash,
+	router: R,
+	table: Arc<SharedTable>,
+	transaction_pool: Arc<Mutex<TransactionPool>>,
 }
 
-impl<C: PolkadotApi, R: TableRouter> bft::Proposer for Proposer<C, R> {
+impl<C, R, P> bft::Proposer for Proposer<C, R, P>
+	where
+		C: PolkadotApi,
+		R: TableRouter,
+		P: Collators,
+{
 	type Error = Error;
-	type Create = Result<SubstrateBlock, Error>;
+	type Create = future::Either<
+		CreateProposal<C, R, P>,
+		future::FutureResult<SubstrateBlock, Error>,
+	>;
 	type Evaluate = Box<Future<Item=bool, Error=Error>>;
 
 	fn propose(&self) -> Self::Create {
-		debug!(target: "bft", "proposing block on top of parent ({}, {:?})", self.parent_number, self.parent_hash);
+		const ATTEMPT_PROPOSE_EVERY: Duration = Duration::from_millis(100);
 
-		// TODO: handle case when current timestamp behind that in state.
-		let mut block_builder = self.client.build_block(
-			&self.parent_id,
-			current_timestamp()
-		)?;
+		let initial_included = self.table.includable_count();
+		let enough_candidates = self.dynamic_inclusion.acceptable_in(
+			Instant::now(),
+			initial_included,
+		).unwrap_or_default();
 
-		let readiness_evaluator = Ready::create(self.parent_id.clone(), &*self.client);
+		let timing = {
+			let delay = self.delay.clone();
+			let dynamic_inclusion = self.dynamic_inclusion.clone();
+			let make_timing = move |handle| -> Result<ProposalTiming, ::std::io::Error> {
+				let attempt_propose = Interval::new(ATTEMPT_PROPOSE_EVERY, handle)?;
+				let enough_candidates = Timeout::new(enough_candidates, handle)?;
+				Ok(ProposalTiming {
+					attempt_propose,
+					enough_candidates,
+					dynamic_inclusion,
+					minimum_delay: Some(delay),
+					last_included: initial_included,
+				})
+			};
 
-		{
-			let mut pool = self.transaction_pool.lock();
-			let mut unqueue_invalid = Vec::new();
-			let mut pending_size = 0;
-			pool.cull(None, readiness_evaluator.clone());
-			for pending in pool.pending(readiness_evaluator.clone()) {
-				// skip and cull transactions which are too large.
-				if pending.encoded_size() > MAX_TRANSACTIONS_SIZE {
-					unqueue_invalid.push(pending.hash().clone());
-					continue
+			match make_timing(&self.handle) {
+				Ok(timing) => timing,
+				Err(e) => {
+					return future::Either::B(future::err(timer_error(&e)));
 				}
-
-				if pending_size + pending.encoded_size() >= MAX_TRANSACTIONS_SIZE { break }
-
-				match block_builder.push_extrinsic(pending.as_transaction().clone()) {
-					Ok(()) => {
-						pending_size += pending.encoded_size();
-					}
-					Err(_) => {
-						unqueue_invalid.push(pending.hash().clone());
-					}
-				}
-			}
-
-			for tx_hash in unqueue_invalid {
-				pool.remove(&tx_hash, false);
-			}
-		}
-
-		let polkadot_block = block_builder.bake();
-		info!("Proposing block [number: {}; hash: {}; parent_hash: {}; extrinsics: [{}]]",
-			polkadot_block.header.number,
-			Hash::from(polkadot_block.header.blake2_256()),
-			polkadot_block.header.parent_hash,
-			polkadot_block.extrinsics.iter()
-				.map(|xt| format!("{}", Hash::from(xt.blake2_256())))
-				.collect::<Vec<_>>()
-				.join(", ")
-		);
-
-		let substrate_block = Slicable::decode(&mut polkadot_block.encode().as_slice())
-			.expect("polkadot blocks defined to serialize to substrate blocks correctly; qed");
-
-		assert!(evaluate_proposal(&substrate_block, &*self.client, current_timestamp(), &self.parent_hash, self.parent_number, &self.parent_id).is_ok());
-
-		Ok(substrate_block)
-	}
-
-	// TODO: certain kinds of errors here should lead to a misbehavior report.
-	fn evaluate(&self, proposal: &SubstrateBlock) -> Self::Evaluate {
-		debug!(target: "bft", "evaluating block on top of parent ({}, {:?})", self.parent_number, self.parent_hash);
-
-		let evaluated = match evaluate_proposal(proposal, &*self.client, current_timestamp(), &self.parent_hash, self.parent_number, &self.parent_id) {
-			Ok(x) => Ok(x),
-			Err(e) => match *e.kind() {
-				ErrorKind::PolkadotApi(polkadot_api::ErrorKind::Executor(_)) => Ok(false),
-				ErrorKind::ProposalNotForPolkadot => Ok(false),
-				ErrorKind::TimestampInFuture => Ok(false),
-				ErrorKind::WrongParentHash(_, _) => Ok(false),
-				ErrorKind::ProposalTooLarge(_) => Ok(false),
-				_ => Err(e),
 			}
 		};
 
-		// delay casting vote until able.
-		Box::new(self.delay.clone().map_err(Error::from).and_then(move |_| evaluated))
+		future::Either::A(CreateProposal {
+			parent_hash: self.parent_hash.clone(),
+			parent_number: self.parent_number.clone(),
+			parent_id: self.parent_id.clone(),
+			client: self.client.clone(),
+			transaction_pool: self.transaction_pool.clone(),
+			collation: CollationFetch::new(
+				self.local_duty.validation,
+				self.parent_id.clone(),
+				self.parent_hash.clone(),
+				self.collators.clone(),
+				self.client.clone()
+			),
+			table: self.table.clone(),
+			router: self.router.clone(),
+			timing,
+		})
+	}
+
+	fn evaluate(&self, proposal: &SubstrateBlock) -> Self::Evaluate {
+		debug!(target: "bft", "evaluating block on top of parent ({}, {:?})", self.parent_number, self.parent_hash);
+
+		let active_parachains = match self.client.active_parachains(&self.parent_id) {
+			Ok(x) => x,
+			Err(e) => return Box::new(future::err(e.into())) as Box<_>,
+		};
+
+		let current_timestamp = current_timestamp();
+
+		// do initial serialization and structural integrity checks.
+		let maybe_proposal = evaluation::evaluate_initial(
+			proposal,
+			current_timestamp,
+			&self.parent_hash,
+			self.parent_number,
+			&active_parachains,
+		);
+
+		let proposal = match maybe_proposal {
+			Ok(p) => p,
+			Err(e) => {
+				// TODO: these errors are easily re-checked in runtime.
+				debug!(target: "bft", "Invalid proposal: {:?}", e);
+				return Box::new(future::ok(false));
+			}
+		};
+
+		let vote_delays = {
+			// delay casting vote until able (according to minimum block time)
+			let minimum_delay = self.delay.clone()
+				.map_err(|e| timer_error(&*e));
+
+			let included_candidate_hashes = proposal
+				.parachain_heads()
+				.iter()
+				.map(|candidate| candidate.hash());
+
+			// delay casting vote until we have proof that all candidates are
+			// includable.
+			let includability_tracker = self.table.track_includability(included_candidate_hashes)
+				.map_err(|_| ErrorKind::PrematureDestruction.into());
+
+			// the duration at which the given number of parachains is acceptable.
+			let count_delay = self.dynamic_inclusion.acceptable_in(
+				Instant::now(),
+				proposal.parachain_heads().len(),
+			);
+
+			// the duration until the given timestamp is current
+			let proposed_timestamp = proposal.timestamp();
+			let timestamp_delay = if proposed_timestamp > current_timestamp {
+				Some(Duration::from_secs(proposed_timestamp - current_timestamp))
+			} else {
+				None
+			};
+
+			// construct a future from the maximum of the two durations.
+			let temporary_delay = match ::std::cmp::max(timestamp_delay, count_delay) {
+				Some(duration) => {
+					let maybe_timeout = Timeout::new(duration, &self.handle);
+
+					let f = future::result(maybe_timeout)
+						.and_then(|timeout| timeout)
+						.map_err(|e| timer_error(&e));
+
+					future::Either::A(f)
+				}
+				None => future::Either::B(future::ok(())),
+			};
+
+			minimum_delay.join3(includability_tracker, temporary_delay)
+		};
+
+		// evaluate whether the block is actually valid.
+		// TODO: is it better to delay this until the delays are finished?
+		let evaluated = self.client.evaluate_block(&self.parent_id, proposal.into()).map_err(Into::into);
+		let future = future::result(evaluated).and_then(move |good| {
+			let end_result = future::ok(good);
+			if good {
+				// delay a "good" vote.
+				future::Either::A(vote_delays.and_then(|_| end_result))
+			} else {
+				// don't delay a "bad" evaluation.
+				future::Either::B(end_result)
+			}
+		});
+
+		Box::new(future) as Box<_>
 	}
 
 	fn round_proposer(&self, round_number: usize, authorities: &[AuthorityId]) -> AuthorityId {
@@ -697,45 +546,186 @@ impl<C: PolkadotApi, R: TableRouter> bft::Proposer for Proposer<C, R> {
 	}
 }
 
-fn evaluate_proposal<C: PolkadotApi>(
-	proposal: &SubstrateBlock,
-	client: &C,
-	now: Timestamp,
-	parent_hash: &HeaderHash,
+fn current_timestamp() -> Timestamp {
+	use std::time;
+
+	time::SystemTime::now().duration_since(time::UNIX_EPOCH)
+		.expect("now always later than unix epoch; qed")
+		.as_secs()
+}
+
+struct ProposalTiming {
+	attempt_propose: Interval,
+	dynamic_inclusion: DynamicInclusion,
+	enough_candidates: Timeout,
+	minimum_delay: Option<Shared<Timeout>>,
+	last_included: usize,
+}
+
+impl ProposalTiming {
+	// whether it's time to attempt a proposal.
+	// shouldn't be called outside of the context of a task.
+	fn poll(&mut self, included: usize) -> Poll<(), Error> {
+		// first drain from the interval so when the minimum delay is up
+		// we don't have any notifications built up.
+		//
+		// this interval is just meant to produce periodic task wakeups
+		// that lead to the `dynamic_inclusion` getting updated as necessary.
+		if let Async::Ready(x) = self.attempt_propose.poll()
+			.map_err(|e| timer_error(&e))?
+		{
+			x.expect("timer still alive; intervals never end; qed");
+		}
+
+		if let Some(ref mut min) = self.minimum_delay {
+			try_ready!(min.poll().map_err(|e| timer_error(&*e)));
+		}
+
+		self.minimum_delay = None; // after this point, the future must have completed.
+
+		if included == self.last_included {
+			return self.enough_candidates.poll().map_err(|e| timer_error(&e));
+		}
+
+		// the amount of includable candidates has changed. schedule a wakeup
+		// if it's not sufficient anymore.
+		let now = Instant::now();
+		match self.dynamic_inclusion.acceptable_in(now, included) {
+			Some(duration) => {
+				self.last_included = included;
+				self.enough_candidates.reset(now + duration);
+				self.enough_candidates.poll().map_err(|e| timer_error(&e))
+			}
+			None => {
+				Ok(Async::Ready(()))
+			}
+		}
+	}
+}
+
+/// Future which resolves upon the creation of a proposal.
+pub struct CreateProposal<C: PolkadotApi, R, P: Collators>  {
+	parent_hash: HeaderHash,
 	parent_number: BlockNumber,
-	parent_id: &C::CheckedBlockId,
-) -> Result<bool, Error> {
-	const MAX_TIMESTAMP_DRIFT: Timestamp = 4;
+	parent_id: C::CheckedBlockId,
+	client: Arc<C>,
+	transaction_pool: Arc<Mutex<TransactionPool>>,
+	collation: CollationFetch<P, C>,
+	router: R,
+	table: Arc<SharedTable>,
+	timing: ProposalTiming,
+}
 
-	let encoded = Slicable::encode(proposal);
-	let proposal = PolkadotGenericBlock::decode(&mut &encoded[..])
-		.and_then(|b| PolkadotBlock::from(b).ok())
-		.ok_or_else(|| ErrorKind::ProposalNotForPolkadot)?;
+impl<C, R, P> CreateProposal<C, R, P>
+	where
+		C: PolkadotApi,
+		R: TableRouter,
+		P: Collators,
+{
+	fn propose_with(&self, candidates: Vec<CandidateReceipt>) -> Result<SubstrateBlock, Error> {
+		// TODO: handle case when current timestamp behind that in state.
+		let timestamp = current_timestamp();
+		let mut block_builder = self.client.build_block(
+			&self.parent_id,
+			timestamp,
+			candidates,
+		)?;
 
-	let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| {
-		a + Slicable::encode(tx).len()
+		let readiness_evaluator = Ready::create(self.parent_id.clone(), &*self.client);
+
+		{
+			let mut pool = self.transaction_pool.lock();
+			let mut unqueue_invalid = Vec::new();
+			let mut pending_size = 0;
+
+			pool.cull(None, readiness_evaluator.clone());
+			for pending in pool.pending(readiness_evaluator.clone()) {
+				// skip and cull transactions which are too large.
+				if pending.encoded_size() > MAX_TRANSACTIONS_SIZE {
+					unqueue_invalid.push(pending.hash().clone());
+					continue
+				}
+
+				if pending_size + pending.encoded_size() >= MAX_TRANSACTIONS_SIZE { break }
+
+				match block_builder.push_extrinsic(pending.as_transaction().clone()) {
+					Ok(()) => {
+						pending_size += pending.encoded_size();
+					}
+					Err(e) => {
+						trace!(target: "transaction-pool", "Invalid transaction: {}", e);
+						unqueue_invalid.push(pending.hash().clone());
+					}
+				}
+			}
+
+			for tx_hash in unqueue_invalid {
+				pool.remove(&tx_hash, false);
+			}
+		}
+
+		let polkadot_block = block_builder.bake();
+
+		info!("Proposing block [number: {}; hash: {}; parent_hash: {}; extrinsics: [{}]]",
+			polkadot_block.header.number,
+			Hash::from(polkadot_block.header.blake2_256()),
+			polkadot_block.header.parent_hash,
+			polkadot_block.extrinsics.iter()
+				.map(|xt| format!("{}", Hash::from(xt.blake2_256())))
+				.collect::<Vec<_>>()
+				.join(", ")
+		);
+
+		let substrate_block = Slicable::decode(&mut polkadot_block.encode().as_slice())
+			.expect("polkadot blocks defined to serialize to substrate blocks correctly; qed");
+
+		// TODO: full re-evaluation
+		let active_parachains = self.client.active_parachains(&self.parent_id)?;
+		assert!(evaluation::evaluate_initial(
+			&substrate_block,
+			timestamp,
+			&self.parent_hash,
+			self.parent_number,
+			&active_parachains,
+		).is_ok());
+
+		Ok(substrate_block)
+	}
+}
+
+impl<C, R, P> Future for CreateProposal<C, R, P>
+	where
+		C: PolkadotApi,
+		R: TableRouter,
+		P: Collators,
+{
+	type Item = SubstrateBlock;
+	type Error = Error;
+
+	fn poll(&mut self) -> Poll<SubstrateBlock, Error> {
+		// 1. poll local collation future.
+		match self.collation.poll() {
+			Ok(Async::Ready((collation, extrinsic))) => {
+				let hash = collation.receipt.hash();
+				self.router.local_candidate_data(hash, collation.block_data, extrinsic);
+
+				// TODO: if we are an availability guarantor also, we should produce an availability statement.
+				self.table.sign_and_import(&self.router, GenericStatement::Candidate(collation.receipt));
+			}
+			Ok(Async::NotReady) => {},
+			Err(_) => {}, // TODO: handle this failure to collate.
+		}
+
+		// 2. try to propose if we have enough includable candidates and other
+		// delays have concluded.
+		let included = self.table.includable_count();
+		try_ready!(self.timing.poll(included));
+
+		// 3. propose
+		let proposed_candidates = self.table.with_proposal(|proposed_set| {
+			proposed_set.into_iter().cloned().collect()
 		});
 
-	if transactions_size > MAX_TRANSACTIONS_SIZE {
-		bail!(ErrorKind::ProposalTooLarge(transactions_size))
+		self.propose_with(proposed_candidates).map(Async::Ready)
 	}
-
-	if proposal.header.parent_hash != *parent_hash {
-		bail!(ErrorKind::WrongParentHash(*parent_hash, proposal.header.parent_hash));
-	}
-
-	if proposal.header.number != parent_number + 1 {
-		bail!(ErrorKind::WrongNumber(parent_number + 1, proposal.header.number))
-	}
-
-	let block_timestamp = proposal.timestamp();
-
-	// TODO: just defer using `tokio_timer` to delay prepare vote.
-	if block_timestamp > now + MAX_TIMESTAMP_DRIFT {
-		bail!(ErrorKind::TimestampInFuture)
-	}
-
-	// execute the block.
-	client.evaluate_block(parent_id, proposal.into())?;
-	Ok(true)
 }

polkadot/consensus/src/service.rs

@@ -22,19 +22,23 @@
 use std::thread;
 use std::time::{Duration, Instant};
 use std::sync::Arc;
-use futures::{future, Future, Stream, Sink, Async, Canceled, Poll};
-use parking_lot::Mutex;
-use substrate_network as net;
-use tokio_core::reactor;
+
+use bft::{self, BftService};
 use client::{BlockchainEvents, ChainHead};
-use runtime_support::Hashable;
+use ed25519;
+use futures::prelude::*;
+use futures::{future, Canceled};
+use parking_lot::Mutex;
+use polkadot_api::PolkadotApi;
+use polkadot_primitives::AccountId;
+use polkadot_primitives::parachain::{Id as ParaId, BlockData, Extrinsic, CandidateReceipt};
 use primitives::{Hash, AuthorityId};
 use primitives::block::{Id as BlockId, HeaderHash, Header};
-use polkadot_primitives::parachain::{BlockData, Extrinsic, CandidateReceipt};
-use polkadot_api::PolkadotApi;
-use bft::{self, BftService};
+use runtime_support::Hashable;
+use substrate_network as net;
+use tokio_core::reactor;
 use transaction_pool::TransactionPool;
-use ed25519;
+
 use super::{TableRouter, SharedTable, ProposerFactory};
 use error;
 
@@ -174,6 +178,15 @@ impl<E> Sink for BftSink<E> {
 
 struct Network(Arc<net::ConsensusService>);
 
+impl super::Network for Network {
+	type TableRouter = Router;
+	fn table_router(&self, _table: Arc<SharedTable>) -> Self::TableRouter {
+		Router {
+			network: self.0.clone()
+		}
+	}
+}
+
 fn start_bft<F, C>(
 	header: &Header,
 	handle: reactor::Handle,
@@ -224,6 +237,7 @@ impl Service {
 		client: Arc<C>,
 		network: Arc<net::ConsensusService>,
 		transaction_pool: Arc<Mutex<TransactionPool>>,
+		parachain_empty_duration: Duration,
 		key: ed25519::Pair,
 	) -> Service
 		where
@@ -233,10 +247,13 @@ impl Service {
 		let thread = thread::spawn(move || {
 			let mut core = reactor::Core::new().expect("tokio::Core could not be created");
 			let key = Arc::new(key);
+
 			let factory = ProposerFactory {
 				client: client.clone(),
 				transaction_pool: transaction_pool.clone(),
 				network: Network(network.clone()),
+				collators: NoCollators,
+				parachain_empty_duration,
 				handle: core.handle(),
 			};
 			let bft_service = Arc::new(BftService::new(client.clone(), key, factory));
@@ -312,17 +329,25 @@ impl Drop for Service {
 	}
 }
 
-impl super::Network for Network {
-	type TableRouter = Router;
-	fn table_router(&self, _table: Arc<SharedTable>) -> Self::TableRouter {
-		Router {
-			network: self.0.clone()
-		}
+// Collators implementation which never collates anything.
+// TODO: do a real implementation.
+#[derive(Clone, Copy)]
+struct NoCollators;
+
+impl ::collation::Collators for NoCollators {
+	type Error = ();
+	type Collation = future::Empty<::collation::Collation, ()>;
+
+	fn collate(&self, _parachain: ParaId, _relay_parent: Hash) -> Self::Collation {
+		future::empty()
 	}
+
+	fn note_bad_collator(&self, _collator: AccountId) { }
 }
 
 type FetchCandidateAdapter = future::Map<net::FetchFuture, fn(Vec<u8>) -> BlockData>;
 
+#[derive(Clone)]
 struct Router {
 	network: Arc<net::ConsensusService>,
 }

polkadot/consensus/src/shared_table/includable.rs

@@ -0,0 +1,137 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Implements a future which resolves when all of the candidates referenced are includable.
+
+use std::collections::HashMap;
+
+use futures::prelude::*;
+use futures::sync::oneshot;
+
+use polkadot_primitives::Hash;
+
+/// Track includability of a set of candidates,
+pub(super) fn track<I: IntoIterator<Item=(Hash, bool)>>(candidates: I) -> (IncludabilitySender, Includable) {
+	let (tx, rx) = oneshot::channel();
+	let tracking: HashMap<_, _> = candidates.into_iter().collect();
+	let includable_count = tracking.values().filter(|x| **x).count();
+
+	let mut sender = IncludabilitySender {
+		tracking,
+		includable_count,
+		sender: Some(tx),
+	};
+
+	sender.try_complete();
+
+	(
+		sender,
+		Includable(rx),
+	)
+}
+
+/// The sending end of the includability sender.
+pub(super) struct IncludabilitySender {
+	tracking: HashMap<Hash, bool>,
+	includable_count: usize,
+	sender: Option<oneshot::Sender<()>>,
+}
+
+impl IncludabilitySender {
+	/// update the inner candidate. wakes up the task as necessary.
+	/// returns `Err(Canceled)` if the other end has hung up.
+	///
+	/// returns `true` when this is completed and should be destroyed.
+	pub fn update_candidate(&mut self, candidate: Hash, includable: bool) -> bool {
+		use std::collections::hash_map::Entry;
+
+		match self.tracking.entry(candidate) {
+			Entry::Vacant(_) => {}
+			Entry::Occupied(mut entry) => {
+				let old = entry.insert(includable);
+				if !old && includable {
+					self.includable_count += 1;
+				} else if old && !includable {
+					self.includable_count -= 1;
+				}
+			}
+		}
+
+		self.try_complete()
+	}
+
+	/// whether the sender is completed.
+	pub fn is_complete(&self) -> bool {
+		self.sender.is_none()
+	}
+
+	fn try_complete(&mut self) -> bool {
+		if self.includable_count == self.tracking.len() {
+			if let Some(sender) = self.sender.take() {
+				let _ = sender.send(());
+			}
+
+			true
+		} else {
+			false
+		}
+	}
+}
+
+/// Future that resolves when all the candidates within are includable.
+pub struct Includable(oneshot::Receiver<()>);
+
+impl Future for Includable {
+	type Item = ();
+	type Error = oneshot::Canceled;
+
+	fn poll(&mut self) -> Poll<(), oneshot::Canceled> {
+		self.0.poll()
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn it_works() {
+		let hash1 = [1; 32].into();
+		let hash2 = [2; 32].into();
+		let hash3 = [3; 32].into();
+
+		let (mut sender, recv) = track([
+			(hash1, true),
+			(hash2, true),
+			(hash2, false), // overwrite should favor latter.
+			(hash3, true),
+		].iter().cloned());
+
+		assert!(!sender.is_complete());
+
+		// true -> false transition is possible and should be handled.
+		sender.update_candidate(hash1, false);
+		assert!(!sender.is_complete());
+
+		sender.update_candidate(hash2, true);
+		assert!(!sender.is_complete());
+
+		sender.update_candidate(hash1, true);
+		assert!(sender.is_complete());
+
+		recv.wait().unwrap();
+	}
+}

polkadot/consensus/src/shared_table/mod.rs

@@ -0,0 +1,566 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Parachain statement table meant to to shared with a message router
+//! and a consensus proposer.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+use table::{self, Table, Context as TableContextTrait};
+use table::generic::Statement as GenericStatement;
+use collation::Collation;
+use polkadot_primitives::Hash;
+use polkadot_primitives::parachain::{Id as ParaId, BlockData, Extrinsic, CandidateReceipt};
+use primitives::AuthorityId;
+
+use parking_lot::Mutex;
+use futures::{future, prelude::*};
+
+use super::{GroupInfo, TableRouter};
+use self::includable::IncludabilitySender;
+
+mod includable;
+
+pub use self::includable::Includable;
+
+struct TableContext {
+	parent_hash: Hash,
+	key: Arc<::ed25519::Pair>,
+	groups: HashMap<ParaId, GroupInfo>,
+}
+
+impl table::Context for TableContext {
+	fn is_member_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
+		self.groups.get(group).map_or(false, |g| g.validity_guarantors.contains(authority))
+	}
+
+	fn is_availability_guarantor_of(&self, authority: &AuthorityId, group: &ParaId) -> bool {
+		self.groups.get(group).map_or(false, |g| g.availability_guarantors.contains(authority))
+	}
+
+	fn requisite_votes(&self, group: &ParaId) -> (usize, usize) {
+		self.groups.get(group).map_or(
+			(usize::max_value(), usize::max_value()),
+			|g| (g.needed_validity, g.needed_availability),
+		)
+	}
+}
+
+impl TableContext {
+	fn local_id(&self) -> AuthorityId {
+		self.key.public().0
+	}
+
+	fn sign_statement(&self, statement: table::Statement) -> table::SignedStatement {
+		let signature = ::sign_table_statement(&statement, &self.key, &self.parent_hash).into();
+		let local_id = self.key.public().0;
+
+		table::SignedStatement {
+			statement,
+			signature,
+			sender: local_id,
+		}
+	}
+}
+
+/// Source of statements
+pub enum StatementSource {
+	/// Locally produced statement.
+	Local,
+	/// Received statement from remote source, with optional sender.
+	Remote(Option<AuthorityId>),
+}
+
+// A shared table object.
+struct SharedTableInner {
+	table: Table<TableContext>,
+	proposed_digest: Option<Hash>,
+	checked_validity: HashSet<Hash>,
+	checked_availability: HashSet<Hash>,
+	trackers: Vec<IncludabilitySender>,
+}
+
+impl SharedTableInner {
+	// Import a single statement. Provide a handle to a table router and a function
+	// used to determine if a referenced candidate is valid.
+	fn import_statement<R: TableRouter, C: FnMut(Collation) -> bool>(
+		&mut self,
+		context: &TableContext,
+		router: &R,
+		statement: table::SignedStatement,
+		statement_source: StatementSource,
+		check_candidate: C,
+	) -> StatementProducer<
+		<R::FetchCandidate as IntoFuture>::Future,
+		<R::FetchExtrinsic as IntoFuture>::Future,
+		C,
+	> {
+		// this blank producer does nothing until we attach some futures
+		// and set a candidate digest.
+		let received_from = match statement_source {
+			StatementSource::Local => return Default::default(),
+			StatementSource::Remote(from) => from,
+		};
+
+		let summary = match self.table.import_statement(context, statement, received_from) {
+			Some(summary) => summary,
+			None => return Default::default(),
+		};
+
+		self.update_trackers(&summary.candidate, context);
+
+		let local_id = context.local_id();
+
+		let is_validity_member = context.is_member_of(&local_id, &summary.group_id);
+		let is_availability_member =
+			context.is_availability_guarantor_of(&local_id, &summary.group_id);
+
+		let digest = &summary.candidate;
+
+		// TODO: consider a strategy based on the number of candidate votes as well.
+		// only check validity if this wasn't locally proposed.
+		let checking_validity = is_validity_member
+			&& self.proposed_digest.as_ref().map_or(true, |d| d != digest)
+			&& self.checked_validity.insert(digest.clone());
+
+		let checking_availability = is_availability_member
+			&& self.checked_availability.insert(digest.clone());
+
+		let work = if checking_validity || checking_availability {
+			match self.table.get_candidate(&digest) {
+				None => None, // TODO: handle table inconsistency somehow?
+				Some(candidate) => {
+					let fetch_block_data =
+						router.fetch_block_data(candidate).into_future().fuse();
+
+					let fetch_extrinsic = if checking_availability {
+						Some(
+							router.fetch_extrinsic_data(candidate).into_future().fuse()
+						)
+					} else {
+						None
+					};
+
+					Some(Work {
+						candidate_receipt: candidate.clone(),
+						fetch_block_data,
+						fetch_extrinsic,
+						evaluate: checking_validity,
+						check_candidate,
+					})
+				}
+			}
+		} else {
+			None
+		};
+
+		StatementProducer {
+			produced_statements: Default::default(),
+			work,
+		}
+	}
+
+	fn update_trackers(&mut self, candidate: &Hash, context: &TableContext) {
+		let includable = self.table.candidate_includable(candidate, context);
+		for i in (0..self.trackers.len()).rev() {
+			if self.trackers[i].update_candidate(candidate.clone(), includable) {
+				self.trackers.swap_remove(i);
+			}
+		}
+	}
+}
+
+/// Produced statements about a specific candidate.
+/// Both may be `None`.
+#[derive(Default)]
+pub struct ProducedStatements {
+	/// A statement about the validity of the candidate.
+	pub validity: Option<table::Statement>,
+	/// A statement about availability of data. If this is `Some`,
+	/// then `block_data` and `extrinsic` should be `Some` as well.
+	pub availability: Option<table::Statement>,
+	/// Block data to ensure availability of.
+	pub block_data: Option<BlockData>,
+	/// Extrinsic data to ensure availability of.
+	pub extrinsic: Option<Extrinsic>,
+}
+
+/// Future that produces statements about a specific candidate.
+pub struct StatementProducer<D: Future, E: Future, C> {
+	produced_statements: ProducedStatements,
+	work: Option<Work<D, E, C>>,
+}
+
+struct Work<D: Future, E: Future, C> {
+	candidate_receipt: CandidateReceipt,
+	fetch_block_data: future::Fuse<D>,
+	fetch_extrinsic: Option<future::Fuse<E>>,
+	evaluate: bool,
+	check_candidate: C
+}
+
+impl<D: Future, E: Future, C> Default for StatementProducer<D, E, C> {
+	fn default() -> Self {
+		StatementProducer {
+			produced_statements: Default::default(),
+			work: None,
+		}
+	}
+}
+
+impl<D, E, C, Err> Future for StatementProducer<D, E, C>
+	where
+		D: Future<Item=BlockData,Error=Err>,
+		E: Future<Item=Extrinsic,Error=Err>,
+		C: FnMut(Collation) -> bool,
+{
+	type Item = ProducedStatements;
+	type Error = Err;
+
+	fn poll(&mut self) -> Poll<ProducedStatements, Err> {
+		let work = match self.work {
+			Some(ref mut work) => work,
+			None => return Ok(Async::Ready(::std::mem::replace(&mut self.produced_statements, Default::default()))),
+		};
+
+		if let Async::Ready(block_data) = work.fetch_block_data.poll()? {
+			self.produced_statements.block_data = Some(block_data.clone());
+			if work.evaluate {
+				let is_good = (work.check_candidate)(Collation {
+					block_data,
+					receipt: work.candidate_receipt.clone(),
+				});
+
+				let hash = work.candidate_receipt.hash();
+				self.produced_statements.validity = Some(if is_good {
+					GenericStatement::Valid(hash)
+				} else {
+					GenericStatement::Invalid(hash)
+				});
+			}
+		}
+
+		if let Some(ref mut fetch_extrinsic) = work.fetch_extrinsic {
+			if let Async::Ready(extrinsic) = fetch_extrinsic.poll()? {
+				self.produced_statements.extrinsic = Some(extrinsic);
+			}
+		}
+
+		let done = self.produced_statements.block_data.is_some() && {
+			if work.evaluate {
+				true
+			} else if self.produced_statements.extrinsic.is_some() {
+				self.produced_statements.availability =
+					Some(GenericStatement::Available(work.candidate_receipt.hash()));
+
+				true
+			} else {
+				false
+			}
+		};
+
+		if done {
+			Ok(Async::Ready(::std::mem::replace(&mut self.produced_statements, Default::default())))
+		} else {
+			Ok(Async::NotReady)
+		}
+	}
+}
+
+/// A shared table object.
+pub struct SharedTable {
+	context: Arc<TableContext>,
+	inner: Arc<Mutex<SharedTableInner>>,
+}
+
+impl Clone for SharedTable {
+	fn clone(&self) -> Self {
+		SharedTable {
+			context: self.context.clone(),
+			inner: self.inner.clone(),
+		}
+	}
+}
+
+impl SharedTable {
+	/// Create a new shared table.
+	///
+	/// Provide the key to sign with, and the parent hash of the relay chain
+	/// block being built.
+	pub fn new(groups: HashMap<ParaId, GroupInfo>, key: Arc<::ed25519::Pair>, parent_hash: Hash) -> Self {
+		SharedTable {
+			context: Arc::new(TableContext { groups, key, parent_hash }),
+			inner: Arc::new(Mutex::new(SharedTableInner {
+				table: Table::default(),
+				proposed_digest: None,
+				checked_validity: HashSet::new(),
+				checked_availability: HashSet::new(),
+				trackers: Vec::new(),
+			}))
+		}
+	}
+
+	/// Get group info.
+	pub fn group_info(&self) -> &HashMap<ParaId, GroupInfo> {
+		&self.context.groups
+	}
+
+	/// Import a single statement. Provide a handle to a table router
+	/// for dispatching any other requests which come up.
+	pub fn import_statement<R: TableRouter, C: FnMut(Collation) -> bool>(
+		&self,
+		router: &R,
+		statement: table::SignedStatement,
+		received_from: StatementSource,
+		check_candidate: C,
+	) -> StatementProducer<<R::FetchCandidate as IntoFuture>::Future, <R::FetchExtrinsic as IntoFuture>::Future, C> {
+		self.inner.lock().import_statement(&*self.context, router, statement, received_from, check_candidate)
+	}
+
+	/// Sign and import a local statement.
+	pub fn sign_and_import<R: TableRouter>(
+		&self,
+		router: &R,
+		statement: table::Statement,
+	) {
+		let proposed_digest = match statement {
+			GenericStatement::Candidate(ref c) => Some(c.hash()),
+			_ => None,
+		};
+
+		let signed_statement = self.context.sign_statement(statement);
+
+		let mut inner = self.inner.lock();
+		if proposed_digest.is_some() {
+			inner.proposed_digest = proposed_digest;
+		}
+
+		let producer = inner.import_statement(
+			&*self.context,
+			router,
+			signed_statement,
+			StatementSource::Local,
+			|_| true,
+		);
+
+		assert!(producer.work.is_none(), "local statement import never leads to additional work; qed");
+	}
+
+	/// Import many statements at once.
+	///
+	/// Provide an iterator yielding pairs of (statement, statement_source).
+	pub fn import_statements<R, I, C, U>(&self, router: &R, iterable: I) -> U
+		where
+			R: TableRouter,
+			I: IntoIterator<Item=(table::SignedStatement, StatementSource, C)>,
+			C: FnMut(Collation) -> bool,
+			U: ::std::iter::FromIterator<StatementProducer<
+				<R::FetchCandidate as IntoFuture>::Future,
+				<R::FetchExtrinsic as IntoFuture>::Future,
+				C,
+			>>,
+	{
+		let mut inner = self.inner.lock();
+
+		iterable.into_iter().map(move |(statement, statement_source, check_candidate)| {
+			inner.import_statement(&*self.context, router, statement, statement_source, check_candidate)
+		}).collect()
+	}
+
+	/// Execute a closure using a specific candidate.
+	///
+	/// Deadlocks if called recursively.
+	pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
+		where F: FnOnce(Option<&CandidateReceipt>) -> U
+	{
+		let inner = self.inner.lock();
+		f(inner.table.get_candidate(digest))
+	}
+
+	/// Execute a closure using the current proposed set.
+	///
+	/// Deadlocks if called recursively.
+	pub fn with_proposal<F, U>(&self, f: F) -> U
+		where F: FnOnce(Vec<&CandidateReceipt>) -> U
+	{
+		let inner = self.inner.lock();
+		f(inner.table.proposed_candidates(&*self.context))
+	}
+
+	/// Get the number of parachains which have available candidates.
+	pub fn includable_count(&self) -> usize {
+		self.inner.lock().table.includable_count()
+	}
+
+	/// Get all witnessed misbehavior.
+	pub fn get_misbehavior(&self) -> HashMap<AuthorityId, table::Misbehavior> {
+		self.inner.lock().table.get_misbehavior().clone()
+	}
+
+	/// Fill a statement batch.
+	pub fn fill_batch<B: table::StatementBatch>(&self, batch: &mut B) {
+		self.inner.lock().table.fill_batch(batch);
+	}
+
+	/// Track includability  of a given set of candidate hashes.
+	pub fn track_includability<I>(&self, iterable: I) -> Includable
+		where I: IntoIterator<Item=Hash>
+	{
+		let mut inner = self.inner.lock();
+
+		let (tx, rx) = includable::track(iterable.into_iter().map(|x| {
+			let includable = inner.table.candidate_includable(&x, &*self.context);
+			(x, includable)
+		}));
+
+		if !tx.is_complete() {
+			inner.trackers.push(tx);
+		}
+
+		rx
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use substrate_keyring::Keyring;
+
+	#[derive(Clone)]
+	struct DummyRouter;
+	impl TableRouter for DummyRouter {
+		type Error = ();
+		type FetchCandidate = ::futures::future::Empty<BlockData,()>;
+		type FetchExtrinsic = ::futures::future::Empty<Extrinsic,()>;
+
+		/// Note local candidate data, making it available on the network to other validators.
+		fn local_candidate_data(&self, _hash: Hash, _block_data: BlockData, _extrinsic: Extrinsic) {
+
+		}
+
+		/// Fetch block data for a specific candidate.
+		fn fetch_block_data(&self, _candidate: &CandidateReceipt) -> Self::FetchCandidate {
+			::futures::future::empty()
+		}
+
+		/// Fetch extrinsic data for a specific candidate.
+		fn fetch_extrinsic_data(&self, _candidate: &CandidateReceipt) -> Self::FetchExtrinsic {
+			::futures::future::empty()
+		}
+	}
+
+	#[test]
+	fn statement_triggers_fetch_and_evaluate() {
+		let mut groups = HashMap::new();
+
+		let para_id = ParaId::from(1);
+		let local_id = Keyring::Alice.to_raw_public();
+		let local_key = Arc::new(Keyring::Alice.pair());
+
+		let validity_other = Keyring::Bob.to_raw_public();
+		let validity_other_key = Keyring::Bob.pair();
+		let parent_hash = Default::default();
+
+		groups.insert(para_id, GroupInfo {
+			validity_guarantors: [local_id, validity_other].iter().cloned().collect(),
+			availability_guarantors: Default::default(),
+			needed_validity: 2,
+			needed_availability: 0,
+		});
+
+		let shared_table = SharedTable::new(groups, local_key.clone(), parent_hash);
+
+		let candidate = CandidateReceipt {
+			parachain_index: para_id,
+			collator: [1; 32],
+			head_data: ::polkadot_primitives::parachain::HeadData(vec![1, 2, 3, 4]),
+			balance_uploads: Vec::new(),
+			egress_queue_roots: Vec::new(),
+			fees: 1_000_000,
+		};
+
+		let candidate_statement = GenericStatement::Candidate(candidate);
+
+		let signature = ::sign_table_statement(&candidate_statement, &validity_other_key, &parent_hash);
+		let signed_statement = ::table::generic::SignedStatement {
+			statement: candidate_statement,
+			signature: signature.into(),
+			sender: validity_other,
+		};
+
+		let producer = shared_table.import_statement(
+			&DummyRouter,
+			signed_statement,
+			StatementSource::Remote(None),
+			|_| true,
+		);
+
+		assert!(producer.work.is_some(), "candidate and local validity group are same");
+		assert!(producer.work.as_ref().unwrap().evaluate, "should evaluate validity");
+	}
+
+	#[test]
+	fn statement_triggers_fetch_and_availability() {
+		let mut groups = HashMap::new();
+
+		let para_id = ParaId::from(1);
+		let local_id = Keyring::Alice.to_raw_public();
+		let local_key = Arc::new(Keyring::Alice.pair());
+
+		let validity_other = Keyring::Bob.to_raw_public();
+		let validity_other_key = Keyring::Bob.pair();
+		let parent_hash = Default::default();
+
+		groups.insert(para_id, GroupInfo {
+			validity_guarantors: [validity_other].iter().cloned().collect(),
+			availability_guarantors: [local_id].iter().cloned().collect(),
+			needed_validity: 1,
+			needed_availability: 1,
+		});
+
+		let shared_table = SharedTable::new(groups, local_key.clone(), parent_hash);
+
+		let candidate = CandidateReceipt {
+			parachain_index: para_id,
+			collator: [1; 32],
+			head_data: ::polkadot_primitives::parachain::HeadData(vec![1, 2, 3, 4]),
+			balance_uploads: Vec::new(),
+			egress_queue_roots: Vec::new(),
+			fees: 1_000_000,
+		};
+
+		let candidate_statement = GenericStatement::Candidate(candidate);
+
+		let signature = ::sign_table_statement(&candidate_statement, &validity_other_key, &parent_hash);
+		let signed_statement = ::table::generic::SignedStatement {
+			statement: candidate_statement,
+			signature: signature.into(),
+			sender: validity_other,
+		};
+
+		let producer = shared_table.import_statement(
+			&DummyRouter,
+			signed_statement,
+			StatementSource::Remote(None),
+			|_| true,
+		);
+
+		assert!(producer.work.is_some(), "candidate and local availability group are same");
+		assert!(producer.work.as_ref().unwrap().fetch_extrinsic.is_some(), "should fetch extrinsic when guaranteeing availability");
+		assert!(!producer.work.as_ref().unwrap().evaluate, "should not evaluate validity");
+	}
+}

polkadot/parachain/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "polkadot-parachain"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Types and utilities for creating and working with parachains"
+
+[dependencies]
+substrate-codec = { path = "../../substrate/codec", default-features = false }
+wasmi = { version = "0.1", optional = true }
+error-chain = { version = "0.11", optional = true }
+
+[dev-dependencies]
+tiny-keccak = "1.4"
+
+[features]
+default = ["std"]
+std = ["substrate-codec/std", "wasmi", "error-chain"]

polkadot/parachain/src/lib.rs

@@ -0,0 +1,146 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Defines primitive types for creating or validating a parachain.
+//!
+//! When compiled with standard library support, this crate exports a `wasm`
+//! module that can be used to validate parachain WASM.
+//!
+//! ## Parachain WASM
+//!
+//! Polkadot parachain WASM is in the form of a module which imports a memory
+//! instance and exports a function `validate`.
+//!
+//! `validate` accepts as input two `i32` values, representing a pointer/length pair
+//! respectively, that encodes `ValidationParams`.
+//!
+//! `validate` returns an `i32` which is a pointer to a little-endian 32-bit integer denoting a length.
+//! Subtracting the length from the initial pointer will give a new pointer to the actual return data,
+//!
+//! ASCII-diagram demonstrating the return data format:
+//!
+//! ```ignore
+//! [return data][len (LE-u32)]
+//!              ^~~returned pointer
+//! ```
+//!
+//! The `load_params` and `write_result` functions provide utilities for setting up
+//! a parachain WASM module in Rust.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+#![cfg_attr(not(feature = "std"), feature(alloc))]
+
+/// Re-export of substrate-codec.
+pub extern crate substrate_codec as codec;
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+
+#[cfg(feature = "std")]
+extern crate core;
+
+#[cfg(feature = "std")]
+extern crate wasmi;
+
+#[cfg(feature = "std")]
+#[macro_use]
+extern crate error_chain;
+
+#[cfg(not(feature = "std"))]
+use alloc::vec::Vec;
+use codec::Slicable;
+
+#[cfg(feature = "std")]
+pub mod wasm;
+
+/// Validation parameters for evaluating the parachain validity function.
+// TODO: consolidated ingress and balance downloads
+#[derive(PartialEq, Eq)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct ValidationParams {
+	/// The collation body.
+	pub block_data: Vec<u8>,
+	/// Previous head-data.
+	pub parent_head: Vec<u8>,
+}
+
+impl Slicable for ValidationParams {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		self.block_data.using_encoded(|s| v.extend(s));
+		self.parent_head.using_encoded(|s| v.extend(s));
+
+		v
+	}
+
+	fn decode<I: codec::Input>(input: &mut I) -> Option<Self> {
+		Some(ValidationParams {
+			block_data: Slicable::decode(input)?,
+			parent_head: Slicable::decode(input)?,
+		})
+	}
+}
+
+/// The result of parachain validation.
+// TODO: egress and balance uploads
+#[derive(PartialEq, Eq)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct ValidationResult {
+	/// New head data that should be included in the relay chain state.
+	pub head_data: Vec<u8>
+}
+
+impl Slicable for ValidationResult {
+	fn encode(&self) -> Vec<u8> {
+		self.head_data.encode()
+	}
+
+	fn decode<I: codec::Input>(input: &mut I) -> Option<Self> {
+		Some(ValidationResult {
+			head_data: Slicable::decode(input)?,
+		})
+	}
+}
+
+/// Load the validation params from memory when implementing a Rust parachain.
+///
+/// Offset and length must have been provided by the validation
+/// function's entry point.
+pub unsafe fn load_params(offset: usize, len: usize) -> ValidationParams {
+	let mut slice = ::core::slice::from_raw_parts(offset as *const u8, len);
+
+	ValidationParams::decode(&mut slice).expect("Invalid input data")
+}
+
+/// Allocate the validation result in memory, getting the return-pointer back.
+///
+/// As described in the crate docs, this is a pointer to the appended length
+/// of the vector.
+pub fn write_result(result: ValidationResult) -> usize {
+	let mut encoded = result.encode();
+	let len = encoded.len();
+
+	assert!(len <= u32::max_value() as usize, "Len too large for parachain-WASM abi");
+	(len as u32).using_encoded(|s| encoded.extend(s));
+
+	// do not alter `encoded` beyond this point. may reallocate.
+	let end_ptr = &encoded[len] as *const u8 as usize;
+
+	// leak so it doesn't get zeroed.
+	::core::mem::forget(encoded);
+	end_ptr
+}

polkadot/parachain/src/wasm.rs

@@ -0,0 +1,166 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! WASM re-execution of a parachain candidate.
+//! In the context of relay-chain candidate evaluation, there are some additional
+//! steps to ensure that the provided input parameters are correct.
+//! Assuming the parameters are correct, this module provides a wrapper around
+//! a WASM VM for re-execution of a parachain candidate.
+
+use codec::Slicable;
+
+use wasmi::{self, Module, ModuleInstance,  MemoryInstance, MemoryDescriptor, MemoryRef, ModuleImportResolver};
+use wasmi::{memory_units, RuntimeValue};
+use wasmi::Error as WasmError;
+
+use super::{ValidationParams, ValidationResult};
+
+use std::cell::RefCell;
+
+error_chain! {
+	types { Error, ErrorKind, ResultExt; }
+	foreign_links {
+		Wasm(WasmError);
+	}
+	errors {
+		/// Call data too big. WASM32 only has a 32-bit address space.
+		ParamsTooLarge(len: usize) {
+			description("Validation parameters took up too much space to execute in WASM"),
+			display("Validation parameters took up {} bytes, max allowed by WASM is {}", len, i32::max_value()),
+		}
+		/// Bad return data or type.
+		BadReturn {
+			description("Validation function returned invalid data."),
+			display("Validation function returned invalid data."),
+		}
+	}
+}
+
+struct Resolver {
+	max_memory: u32, // in pages.
+	memory: RefCell<Option<MemoryRef>>,
+}
+
+impl ModuleImportResolver for Resolver {
+	fn resolve_memory(
+		&self,
+		field_name: &str,
+		descriptor: &MemoryDescriptor,
+	) -> Result<MemoryRef, WasmError> {
+		if field_name == "memory" {
+			let effective_max = descriptor.maximum().unwrap_or(self.max_memory);
+			if descriptor.initial() > self.max_memory || effective_max > self.max_memory {
+				Err(WasmError::Instantiation("Module requested too much memory".to_owned()))
+			} else {
+				let mem = MemoryInstance::alloc(
+					memory_units::Pages(descriptor.initial() as usize),
+					descriptor.maximum().map(|x| memory_units::Pages(x as usize)),
+				)?;
+				*self.memory.borrow_mut() = Some(mem.clone());
+				Ok(mem)
+			}
+		} else {
+			Err(WasmError::Instantiation("Memory imported under unknown name".to_owned()))
+		}
+	}
+}
+
+/// Validate a candidate under the given validation code.
+///
+/// This will fail if the validation code is not a proper parachain validation module.
+pub fn validate_candidate(validation_code: &[u8], params: ValidationParams) -> Result<ValidationResult, Error> {
+	use wasmi::LINEAR_MEMORY_PAGE_SIZE;
+
+	// maximum memory in bytes
+	const MAX_MEM: u32 = 1024 * 1024 * 1024; // 1 GiB
+
+	// instantiate the module.
+	let (module, memory) = {
+		let module = Module::from_buffer(validation_code)?;
+
+		let module_resolver = Resolver {
+			max_memory: MAX_MEM / LINEAR_MEMORY_PAGE_SIZE.0 as u32,
+			memory: RefCell::new(None),
+		};
+
+		let module = ModuleInstance::new(
+			&module,
+			&wasmi::ImportsBuilder::new().with_resolver("env", &module_resolver),
+		)?.run_start(&mut wasmi::NopExternals).map_err(WasmError::Trap)?;
+
+		let memory = module_resolver.memory.borrow_mut()
+			.as_ref()
+			.ok_or_else(|| WasmError::Instantiation("No imported memory instance".to_owned()))?
+			.clone();
+
+		(module, memory)
+	};
+
+	// allocate call data in memory.
+	let (offset, len) = {
+		let encoded_call_data = params.encode();
+
+		// hard limit from WASM.
+		if encoded_call_data.len() > i32::max_value() as usize {
+			bail!(ErrorKind::ParamsTooLarge(encoded_call_data.len()));
+		}
+
+		let call_data_pages = (encoded_call_data.len() / LINEAR_MEMORY_PAGE_SIZE.0) +
+			(encoded_call_data.len() % LINEAR_MEMORY_PAGE_SIZE.0);
+
+		let call_data_pages = wasmi::memory_units::Pages(call_data_pages);
+
+		if memory.current_size() < call_data_pages {
+			memory.grow(call_data_pages - memory.current_size())?;
+		}
+
+		memory.set(0, &encoded_call_data).expect("enough memory allocated just before this; \
+			copying never fails if memory is large enough; qed");
+
+		(0, encoded_call_data.len() as i32)
+	};
+
+	let output = module.invoke_export(
+		"validate",
+		&[RuntimeValue::I32(offset), RuntimeValue::I32(len)],
+		&mut wasmi::NopExternals,
+	)?;
+
+	match output {
+		Some(RuntimeValue::I32(len_offset)) => {
+			let len_offset = len_offset as u32;
+
+			let mut len_bytes = [0u8; 4];
+			memory.get_into(len_offset, &mut len_bytes)?;
+
+			let len = u32::decode(&mut &len_bytes[..])
+				.ok_or_else(|| ErrorKind::BadReturn)?;
+
+			let return_offset = if len > len_offset {
+				bail!(ErrorKind::BadReturn);
+			} else {
+				len_offset - len
+			};
+
+			// TODO: optimize when `wasmi` lets you inspect memory with a closure.
+			let raw_return = memory.get(return_offset, len as usize)?;
+			ValidationResult::decode(&mut &raw_return[..])
+				.ok_or_else(|| ErrorKind::BadReturn)
+				.map_err(Into::into)
+		}
+		_ => bail!(ErrorKind::BadReturn),
+	}
+}

polkadot/parachain/test-chains/.gitignore

@@ -0,0 +1,2 @@
+target/
+Cargo.lock

polkadot/parachain/test-chains/basic_add/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "basic_add"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+description = "Test parachain which adds to a number as its state transition"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+polkadot-parachain = { path = "../../", default-features = false }
+wee_alloc = "0.2.0"
+tiny-keccak = "1.4"
+pwasm-libc = "0.2"
+
+[features]
+default = ["std"]
+std = ["polkadot-parachain/std"]
+
+[profile.release]
+panic = "abort"
+lto = true
+
+[workspace]
+members = []

polkadot/parachain/test-chains/basic_add/src/lib.rs

@@ -0,0 +1,143 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+#![cfg_attr(not(feature = "std"), feature(alloc, core_intrinsics, global_allocator, lang_items))]
+
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+
+extern crate polkadot_parachain as parachain;
+extern crate wee_alloc;
+extern crate tiny_keccak;
+extern crate pwasm_libc;
+
+use parachain::codec::{Slicable, Input};
+
+#[cfg(not(feature = "std"))]
+mod wasm;
+
+#[cfg(not(feature = "std"))]
+pub use wasm::*;
+
+#[cfg(not(feature = "std"))]
+use alloc::vec::Vec;
+
+// Define global allocator.
+#[cfg(not(feature = "std"))]
+#[global_allocator]
+static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
+
+// Head data for this parachain.
+#[derive(Default, Clone)]
+struct HeadData {
+	// Block number
+	number: u64,
+	// parent block keccak256
+	parent_hash: [u8; 32],
+	// hash of post-execution state.
+	post_state: [u8; 32],
+}
+
+impl Slicable for HeadData {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		self.number.using_encoded(|s| v.extend(s));
+		self.parent_hash.using_encoded(|s| v.extend(s));
+		self.post_state.using_encoded(|s| v.extend(s));
+
+		v
+	}
+
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(HeadData {
+			number: Slicable::decode(input)?,
+			parent_hash: Slicable::decode(input)?,
+			post_state: Slicable::decode(input)?,
+		})
+	}
+}
+
+// Block data for this parachain.
+#[derive(Default, Clone)]
+struct BlockData {
+	// State to begin from.
+	state: u64,
+	// Amount to add (overflowing)
+	add: u64,
+}
+
+impl Slicable for BlockData {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		self.state.using_encoded(|s| v.extend(s));
+		self.add.using_encoded(|s| v.extend(s));
+
+		v
+	}
+
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(BlockData {
+			state: Slicable::decode(input)?,
+			add: Slicable::decode(input)?,
+		})
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use parachain::ValidationParams;
+
+	const TEST_CODE: &[u8] = include_bytes!("../wasm/test.wasm");
+
+	fn hash_state(state: u64) -> [u8; 32] {
+		::tiny_keccak::keccak256(state.encode().as_slice())
+	}
+
+	fn hash_head(head: &HeadData) -> [u8; 32] {
+		::tiny_keccak::keccak256(head.encode().as_slice())
+	}
+
+	#[test]
+	fn execute_good_on_parent() {
+		let parent_head = HeadData {
+			number: 0,
+			parent_hash: [0; 32],
+			post_state: hash_state(0),
+		};
+
+		let block_data = BlockData {
+			state: 0,
+			add: 512,
+		};
+
+		let ret = parachain::wasm::validate_candidate(TEST_CODE, ValidationParams {
+			parent_head: parent_head.encode(),
+			block_data: block_data.encode(),
+		}).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data[..]).unwrap();
+
+		assert_eq!(new_head.number, 1);
+		assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+		assert_eq!(new_head.post_state, hash_state(512));
+	}
+}

polkadot/parachain/test-chains/basic_add/src/src

@@ -0,0 +1 @@
+src

polkadot/parachain/test-chains/basic_add/src/wasm.rs

@@ -0,0 +1,58 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Defines WASM module logic.
+
+use parachain::{self, ValidationResult};
+use parachain::codec::Slicable;
+use super::{HeadData, BlockData};
+
+#[lang = "panic_fmt"]
+#[no_mangle]
+pub extern fn panic_fmt(
+	_args: ::core::fmt::Arguments,
+	_file: &'static str,
+	_line: u32,
+	_col: u32,
+) -> ! {
+	use core::intrinsics;
+	unsafe {
+		intrinsics::abort();
+	}
+}
+
+#[no_mangle]
+pub extern fn validate(offset: usize, len: usize) -> usize {
+	let hash_state = |state: u64| ::tiny_keccak::keccak256(state.encode().as_slice());
+
+	let params = unsafe { ::parachain::load_params(offset, len) };
+	let parent_head = HeadData::decode(&mut &params.parent_head[..])
+		.expect("invalid parent head format.");
+
+	let block_data = BlockData::decode(&mut &params.block_data[..])
+		.expect("invalid block data format.");
+
+	assert_eq!(hash_state(block_data.state), parent_head.post_state, "wrong post-state proof");
+	let new_state = block_data.state.saturating_add(block_data.add);
+
+	let new_head = HeadData {
+		number: parent_head.number + 1,
+		parent_hash: ::tiny_keccak::keccak256(&params.parent_head[..]),
+		post_state: hash_state(new_state),
+	};
+
+	parachain::write_result(ValidationResult { head_data: new_head.encode() })
+}

polkadot/parachain/test-chains/build_test_chains.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -e
+
+rm -rf ./target
+for i in */
+do
+  i=${i%/}
+  cd $i
+
+  # TODO: stop using exact nightly when wee-alloc works on normal nightly.
+  RUSTFLAGS="-C link-arg=--import-memory" cargo +nightly-2018-03-07 build --target=wasm32-unknown-unknown --release --no-default-features
+  wasm-gc target/wasm32-unknown-unknown/release/$i.wasm ../../tests/res/$i.wasm
+  cd ..
+done

polkadot/parachain/tests/basic_add.rs

@@ -0,0 +1,170 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Basic parachain that adds a number as part of its state.
+
+extern crate polkadot_parachain as parachain;
+extern crate tiny_keccak;
+
+use parachain::ValidationParams;
+use parachain::codec::{Slicable, Input};
+
+// Head data for this parachain.
+#[derive(Default, Clone)]
+struct HeadData {
+	// Block number
+	number: u64,
+	// parent block keccak256
+	parent_hash: [u8; 32],
+	// hash of post-execution state.
+	post_state: [u8; 32],
+}
+
+impl Slicable for HeadData {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		self.number.using_encoded(|s| v.extend(s));
+		self.parent_hash.using_encoded(|s| v.extend(s));
+		self.post_state.using_encoded(|s| v.extend(s));
+
+		v
+	}
+
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(HeadData {
+			number: Slicable::decode(input)?,
+			parent_hash: Slicable::decode(input)?,
+			post_state: Slicable::decode(input)?,
+		})
+	}
+}
+
+// Block data for this parachain.
+#[derive(Default, Clone)]
+struct BlockData {
+	// State to begin from.
+	state: u64,
+	// Amount to add (overflowing)
+	add: u64,
+}
+
+impl Slicable for BlockData {
+	fn encode(&self) -> Vec<u8> {
+		let mut v = Vec::new();
+
+		self.state.using_encoded(|s| v.extend(s));
+		self.add.using_encoded(|s| v.extend(s));
+
+		v
+	}
+
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		Some(BlockData {
+			state: Slicable::decode(input)?,
+			add: Slicable::decode(input)?,
+		})
+	}
+}
+
+const TEST_CODE: &[u8] = include_bytes!("res/basic_add.wasm");
+
+fn hash_state(state: u64) -> [u8; 32] {
+	::tiny_keccak::keccak256(state.encode().as_slice())
+}
+
+fn hash_head(head: &HeadData) -> [u8; 32] {
+	::tiny_keccak::keccak256(head.encode().as_slice())
+}
+
+#[test]
+fn execute_good_on_parent() {
+	let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 0,
+		add: 512,
+	};
+
+	let ret = parachain::wasm::validate_candidate(TEST_CODE, ValidationParams {
+		parent_head: parent_head.encode(),
+		block_data: block_data.encode(),
+	}).unwrap();
+
+	let new_head = HeadData::decode(&mut &ret.head_data[..]).unwrap();
+
+	assert_eq!(new_head.number, 1);
+	assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+	assert_eq!(new_head.post_state, hash_state(512));
+}
+
+#[test]
+fn execute_good_chain_on_parent() {
+	let mut number = 0;
+	let mut parent_hash = [0; 32];
+	let mut last_state = 0;
+
+	for add in 0..10 {
+		let parent_head = HeadData {
+			number,
+			parent_hash,
+			post_state: hash_state(last_state),
+		};
+
+		let block_data = BlockData {
+			state: last_state,
+			add,
+		};
+
+		let ret = parachain::wasm::validate_candidate(TEST_CODE, ValidationParams {
+			parent_head: parent_head.encode(),
+			block_data: block_data.encode(),
+		}).unwrap();
+
+		let new_head = HeadData::decode(&mut &ret.head_data[..]).unwrap();
+
+		assert_eq!(new_head.number, number + 1);
+		assert_eq!(new_head.parent_hash, hash_head(&parent_head));
+		assert_eq!(new_head.post_state, hash_state(last_state + add));
+
+		number += 1;
+		parent_hash = hash_head(&new_head);
+		last_state += add;
+	}
+}
+
+#[test]
+fn execute_bad_on_parent() {
+		let parent_head = HeadData {
+		number: 0,
+		parent_hash: [0; 32],
+		post_state: hash_state(0),
+	};
+
+	let block_data = BlockData {
+		state: 256, // start state is wrong.
+		add: 256,
+	};
+
+	let _ret = parachain::wasm::validate_candidate(TEST_CODE, ValidationParams {
+		parent_head: parent_head.encode(),
+		block_data: block_data.encode(),
+	}).unwrap_err();
+}

polkadot/primitives/src/lib.rs

@@ -36,7 +36,6 @@ extern crate substrate_serializer;
 extern crate substrate_codec as codec;
 
 pub mod parachain;
-pub mod validator;
 
 /// Virtual account ID that represents the idea of a dispatch/statement being signed by everybody
 /// (who matters). Essentially this means that a majority of validators have decided it is

polkadot/primitives/src/parachain.rs

@@ -37,6 +37,13 @@ impl From<u32> for Id {
 	fn from(x: u32) -> Self { Id(x) }
 }
 
+impl Id {
+	/// Convert this Id into its inner representation.
+	pub fn into_inner(self) -> u32 {
+		self.0
+	}
+}
+
 impl Slicable for Id {
 	fn decode<I: Input>(input: &mut I) -> Option<Self> {
 		u32::decode(input).map(Id)
@@ -86,8 +93,6 @@ impl Slicable for Chain {
 	}
 }
 
-
-
 /// The duty roster specifying what jobs each validator must do.
 #[derive(Clone, PartialEq)]
 #[cfg_attr(feature = "std", derive(Default, Debug))]

polkadot/primitives/src/validator.rs

@@ -1,76 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Validator primitives.
-
-#[cfg(feature = "std")]
-use primitives::bytes;
-use rstd::vec::Vec;
-use parachain;
-
-/// Parachain outgoing message.
-#[derive(PartialEq, Eq)]
-#[cfg_attr(feature = "std", derive(Serialize, Debug))]
-pub struct EgressPost(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
-
-/// Balance upload.
-#[derive(PartialEq, Eq)]
-#[cfg_attr(feature = "std", derive(Serialize, Debug))]
-pub struct BalanceUpload(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
-
-/// Balance download.
-#[derive(PartialEq, Eq)]
-#[cfg_attr(feature = "std", derive(Serialize, Debug))]
-pub struct BalanceDownload(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
-
-/// The result of parachain validation.
-#[derive(PartialEq, Eq)]
-#[cfg_attr(feature = "std", derive(Serialize, Debug))]
-#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))]
-#[cfg_attr(feature = "std", serde(deny_unknown_fields))]
-pub struct ValidationResult {
-	/// New head data that should be included in the relay chain state.
-	pub head_data: parachain::HeadData,
-	/// Outgoing messages (a vec for each parachain).
-	pub egress_queues: Vec<Vec<EgressPost>>,
-	/// Balance uploads
-	pub balance_uploads: Vec<BalanceUpload>,
-}
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-	use substrate_serializer as ser;
-
-	#[test]
-	fn test_validation_result() {
-		assert_eq!(ser::to_string_pretty(&ValidationResult {
-	head_data: parachain::HeadData(vec![1]),
-	egress_queues: vec![vec![EgressPost(vec![1])]],
-	balance_uploads: vec![BalanceUpload(vec![2])],
-			}), r#"{
-  "headData": "0x01",
-  "egressQueues": [
-    [
-      "0x01"
-    ]
-  ],
-  "balanceUploads": [
-    "0x02"
-  ]
-}"#);
-	}
-}

polkadot/runtime/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 rustc-hex = "1.0"
-hex-literal = "0.1.0"
 log = { version = "0.3", optional = true }
 serde = { version = "1.0", default_features = false }
 serde_derive = { version = "1.0", optional = true }
@@ -28,6 +27,9 @@ substrate-runtime-system = { path = "../../substrate/runtime/system" }
 substrate-runtime-timestamp = { path = "../../substrate/runtime/timestamp" }
 polkadot-primitives = { path = "../primitives" }
 
+[dev-dependencies]
+hex-literal = "0.1.0"
+
 [features]
 default = ["std"]
 std = [

polkadot/runtime/src/lib.rs

@@ -27,7 +27,7 @@ extern crate substrate_runtime_support as runtime_support;
 #[macro_use]
 extern crate substrate_runtime_primitives as runtime_primitives;
 
-#[cfg(feature = "std")]
+#[cfg(test)]
 #[macro_use]
 extern crate hex_literal;
 
@@ -55,7 +55,19 @@ use runtime_io::BlakeTwo256;
 use polkadot_primitives::{AccountId, Balance, BlockNumber, Hash, Index, Log, SessionKey, Signature};
 use runtime_primitives::generic;
 use runtime_primitives::traits::{Identity, HasPublicAux};
-#[cfg(feature = "std")] pub use runtime_primitives::BuildExternalities;
+
+#[cfg(feature = "std")]
+pub use runtime_primitives::BuildExternalities;
+
+pub use consensus::Call as ConsensusCall;
+pub use timestamp::Call as TimestampCall;
+pub use parachains::Call as ParachainsCall;
+
+
+/// The position of the timestamp set extrinsic.
+pub const TIMESTAMP_SET_POSITION: u32 = 0;
+/// The position of the parachains set extrinsic.
+pub const PARACHAINS_SET_POSITION: u32 = 1;
 
 /// Concrete runtime type used to parameterize the various modules.
 pub struct Concrete;
@@ -77,19 +89,18 @@ impl system::Trait for Concrete {
 pub type System = system::Module<Concrete>;
 
 impl consensus::Trait for Concrete {
-	type PublicAux = <Self as HasPublicAux>::PublicAux;
+	type PublicAux = <Concrete as HasPublicAux>::PublicAux;
 	type SessionKey = SessionKey;
 }
 /// Consensus module for this concrete runtime.
 pub type Consensus = consensus::Module<Concrete>;
-pub use consensus::Call as ConsensusCall;
 
 impl timestamp::Trait for Concrete {
+	const SET_POSITION: u32 = TIMESTAMP_SET_POSITION;
 	type Value = u64;
 }
 /// Timestamp module for this concrete runtime.
 pub type Timestamp = timestamp::Module<Concrete>;
-pub use timestamp::Call as TimestampCall;
 
 impl session::Trait for Concrete {
 	type ConvertAccountIdToSessionKey = Identity;
@@ -116,7 +127,11 @@ pub type Council = council::Module<Concrete>;
 /// Council voting module for this concrete runtime.
 pub type CouncilVoting = council::voting::Module<Concrete>;
 
-impl parachains::Trait for Concrete {}
+impl parachains::Trait for Concrete {
+	const SET_POSITION: u32 = PARACHAINS_SET_POSITION;
+
+	type PublicAux = <Concrete as HasPublicAux>::PublicAux;
+}
 pub type Parachains = parachains::Module<Concrete>;
 
 impl_outer_dispatch! {
@@ -128,6 +143,7 @@ impl_outer_dispatch! {
 		Democracy = 5,
 		Council = 6,
 		CouncilVoting = 7,
+		Parachains = 8,
 	}
 
 	pub enum PrivCall {

polkadot/runtime/src/parachains.rs

@@ -17,44 +17,70 @@
 //! Main parachains logic. For now this is just the determination of which validators do what.
 
 use polkadot_primitives;
-#[cfg(any(feature = "std", test))] use {runtime_io, runtime_primitives};
 use rstd::prelude::*;
-#[cfg(any(feature = "std", test))] use rstd::marker::PhantomData;
 use codec::{Slicable, Joiner};
 use runtime_support::Hashable;
-#[cfg(any(feature = "std", test))] use runtime_support::StorageValue;
-use runtime_primitives::traits::Executable;
-use polkadot_primitives::parachain::{Id, Chain, DutyRoster};
+
+use runtime_primitives::traits::{Executable, RefInto, MaybeEmpty};
+use polkadot_primitives::parachain::{Id, Chain, DutyRoster, CandidateReceipt};
 use {system, session};
 
-pub trait Trait: system::Trait<Hash = polkadot_primitives::Hash> + session::Trait {}
+use runtime_support::{StorageValue, StorageMap};
+
+#[cfg(any(feature = "std", test))]
+use rstd::marker::PhantomData;
+
+#[cfg(any(feature = "std", test))]
+use {runtime_io, runtime_primitives};
+
+pub trait Trait: system::Trait<Hash = polkadot_primitives::Hash> + session::Trait {
+	/// The position of the set_heads call in the block.
+	const SET_POSITION: u32;
+
+	type PublicAux: RefInto<Self::AccountId> + MaybeEmpty;
+}
 
 decl_module! {
 	pub struct Module<T: Trait>;
+	pub enum Call where aux: <T as Trait>::PublicAux {
+		// provide candidate receipts for parachains, in ascending order by id.
+		fn set_heads(aux, heads: Vec<CandidateReceipt>) = 0;
+	}
 }
 
 decl_storage! {
-	pub trait Store for Module<T: Trait>;
-	// The number of parachains registered at present.
-	pub Count get(count): b"para:count" => default u32;
+	trait Store for Module<T: Trait>;
+	// Vector of all parachain IDs.
+	pub Parachains get(active_parachains): b"para:chains" => default Vec<Id>;
+	// The parachains registered at present.
+	pub Code get(parachain_code): b"para:code" => map [ Id => Vec<u8> ];
+	// The heads of the parachains registered at present. these are kept sorted.
+	pub Heads get(parachain_head): b"para:head" => map [ Id => Vec<u8> ];
+
+	// Did the parachain heads get updated in this block?
+	DidUpdate: b"para:did" => default bool;
 }
 
 impl<T: Trait> Module<T> {
-	/// Calculate the current block's duty roster.
+	/// Calculate the current block's duty roster using system's random seed.
 	pub fn calculate_duty_roster() -> DutyRoster {
-		let parachain_count = Self::count();
-		let validator_count = <session::Module<T>>::validator_count();
+		let parachains = Self::active_parachains();
+		let parachain_count = parachains.len();
+		let validator_count = <session::Module<T>>::validator_count() as usize;
 		let validators_per_parachain = if parachain_count != 0 { (validator_count - 1) / parachain_count } else { 0 };
 
 		let mut roles_val = (0..validator_count).map(|i| match i {
-			i if i < parachain_count * validators_per_parachain =>
-				Chain::Parachain(Id::from(i / validators_per_parachain as u32)),
+			i if i < parachain_count * validators_per_parachain => {
+				let idx = i / validators_per_parachain;
+				Chain::Parachain(parachains[idx].clone())
+			}
 			_ => Chain::Relay,
 		}).collect::<Vec<_>>();
+
 		let mut roles_gua = roles_val.clone();
 
-		let h = <system::Module<T>>::random_seed();
-		let mut seed = h.to_vec().and(b"validator_role_pairs").blake2_256();
+		let random_seed = system::Module::<T>::random_seed();
+		let mut seed = random_seed.to_vec().and(b"validator_role_pairs").blake2_256();
 
 		// shuffle
 		for i in 0..(validator_count - 1) {
@@ -83,16 +109,76 @@ impl<T: Trait> Module<T> {
 			guarantor_duty: roles_gua,
 		}
 	}
+
+	/// Register a parachain with given code.
+	/// Fails if given ID is already used.
+	pub fn register_parachain(id: Id, code: Vec<u8>, initial_head_data: Vec<u8>) {
+		let mut parachains = Self::active_parachains();
+		match parachains.binary_search(&id) {
+			Ok(_) => panic!("Parachain with id {} already exists", id.into_inner()),
+			Err(idx) => parachains.insert(idx, id),
+		}
+
+		<Code<T>>::insert(id, code);
+		<Parachains<T>>::put(parachains);
+		<Heads<T>>::insert(id, initial_head_data);
+	}
+
+	/// Deregister a parachain with given id
+	pub fn deregister_parachain(id: Id) {
+		let mut parachains = Self::active_parachains();
+		match parachains.binary_search(&id) {
+			Ok(idx) => { parachains.remove(idx); }
+			Err(_) => {}
+		}
+
+		<Code<T>>::remove(id);
+		<Heads<T>>::remove(id);
+		<Parachains<T>>::put(parachains);
+	}
+
+	fn set_heads(aux: &<T as Trait>::PublicAux, heads: Vec<CandidateReceipt>) {
+		assert!(aux.is_empty());
+		assert!(!<DidUpdate<T>>::exists(), "Parachain heads must be updated only once in the block");
+		assert!(
+			<system::Module<T>>::extrinsic_index() == T::SET_POSITION,
+			"Parachain heads update extrinsic must be at position {} in the block",
+			T::SET_POSITION
+		);
+
+		let active_parachains = Self::active_parachains();
+		let mut iter = active_parachains.iter();
+
+		// perform this check before writing to storage.
+		for head in &heads {
+			assert!(
+				iter.find(|&p| p == &head.parachain_index).is_some(),
+				"Submitted candidate for unregistered or out-of-order parachain {}",
+				head.parachain_index.into_inner()
+			);
+		}
+
+		for head in heads {
+			let id = head.parachain_index.clone();
+			<Heads<T>>::insert(id, head.head_data.0);
+		}
+
+		<DidUpdate<T>>::put(true);
+	}
 }
 
 impl<T: Trait> Executable for Module<T> {
 	fn execute() {
+		assert!(<Self as Store>::DidUpdate::take(), "Parachain heads must be updated once in the block");
 	}
 }
 
+/// Parachains module genesis configuration.
 #[cfg(any(feature = "std", test))]
 pub struct GenesisConfig<T: Trait> {
-	pub count: u32,
+	/// The initial parachains, mapped to code.
+	pub parachains: Vec<(Id, Vec<u8>)>,
+	/// Phantom data.
 	pub phantom: PhantomData<T>,
 }
 
@@ -100,7 +186,7 @@ pub struct GenesisConfig<T: Trait> {
 impl<T: Trait> Default for GenesisConfig<T> {
 	fn default() -> Self {
 		GenesisConfig {
-			count: 0,
+			parachains: Vec::new(),
 			phantom: PhantomData,
 		}
 	}
@@ -109,12 +195,26 @@ impl<T: Trait> Default for GenesisConfig<T> {
 #[cfg(any(feature = "std", test))]
 impl<T: Trait> runtime_primitives::BuildExternalities for GenesisConfig<T>
 {
-	fn build_externalities(self) -> runtime_io::TestExternalities {
+	fn build_externalities(mut self) -> runtime_io::TestExternalities {
+		use std::collections::HashMap;
 		use runtime_io::twox_128;
 		use codec::Slicable;
-		map![
-			twox_128(<Count<T>>::key()).to_vec() => self.count.encode()
-		]
+
+		self.parachains.sort_unstable_by_key(|&(ref id, _)| id.clone());
+		self.parachains.dedup_by_key(|&mut (ref id, _)| id.clone());
+
+		let only_ids: Vec<_> = self.parachains.iter().map(|&(ref id, _)| id).cloned().collect();
+
+		let mut map: HashMap<_, _> = map![
+			twox_128(<Parachains<T>>::key()).to_vec() => only_ids.encode()
+		];
+
+		for (id, code) in self.parachains {
+			let key = twox_128(&<Code<T>>::key_for(&id)).to_vec();
+			map.insert(key, code.encode());
+		}
+
+		map.into()
 	}
 }
 
@@ -148,12 +248,15 @@ mod tests {
 	impl session::Trait for Test {
 		type ConvertAccountIdToSessionKey = Identity;
 	}
-	impl Trait for Test {}
+	impl Trait for Test {
+		const SET_POSITION: u32 = 0;
+
+		type PublicAux = <Self as HasPublicAux>::PublicAux;
+	}
 
-	type System = system::Module<Test>;
 	type Parachains = Module<Test>;
 
-	fn new_test_ext() -> runtime_io::TestExternalities {
+	fn new_test_ext(parachains: Vec<(Id, Vec<u8>)>) -> runtime_io::TestExternalities {
 		let mut t = system::GenesisConfig::<Test>::default().build_externalities();
 		t.extend(consensus::GenesisConfig::<Test>{
 			code: vec![],
@@ -164,22 +267,59 @@ mod tests {
 			validators: vec![1, 2, 3, 4, 5, 6, 7, 8],
 		}.build_externalities());
 		t.extend(GenesisConfig::<Test>{
-			count: 2,
+			parachains: parachains,
 			phantom: PhantomData,
 		}.build_externalities());
 		t
 	}
 
 	#[test]
-	fn simple_setup_should_work() {
-		with_externalities(&mut new_test_ext(), || {
-			assert_eq!(Parachains::count(), 2);
+	fn active_parachains_should_work() {
+		let parachains = vec![
+			(5u32.into(), vec![1,2,3]),
+			(100u32.into(), vec![4,5,6]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
+			assert_eq!(Parachains::active_parachains(), vec![5u32.into(), 100u32.into()]);
+			assert_eq!(Parachains::parachain_code(&5u32.into()), Some(vec![1,2,3]));
+			assert_eq!(Parachains::parachain_code(&100u32.into()), Some(vec![4,5,6]));
 		});
 	}
 
 	#[test]
-	fn should_work() {
-		with_externalities(&mut new_test_ext(), || {
+	fn register_deregister() {
+		let parachains = vec![
+			(5u32.into(), vec![1,2,3]),
+			(100u32.into(), vec![4,5,6]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
+			assert_eq!(Parachains::active_parachains(), vec![5u32.into(), 100u32.into()]);
+
+			assert_eq!(Parachains::parachain_code(&5u32.into()), Some(vec![1,2,3]));
+			assert_eq!(Parachains::parachain_code(&100u32.into()), Some(vec![4,5,6]));
+
+			Parachains::register_parachain(99u32.into(), vec![7,8,9], vec![1, 1, 1]);
+
+			assert_eq!(Parachains::active_parachains(), vec![5u32.into(), 99u32.into(), 100u32.into()]);
+			assert_eq!(Parachains::parachain_code(&99u32.into()), Some(vec![7,8,9]));
+
+			Parachains::deregister_parachain(5u32.into());
+
+			assert_eq!(Parachains::active_parachains(), vec![99u32.into(), 100u32.into()]);
+			assert_eq!(Parachains::parachain_code(&5u32.into()), None);
+		});
+	}
+
+	#[test]
+	fn duty_roster_works() {
+		let parachains = vec![
+			(0u32.into(), vec![]),
+			(1u32.into(), vec![]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
 			let check_roster = |duty_roster: &DutyRoster| {
 				assert_eq!(duty_roster.validator_duty.len(), 8);
 				assert_eq!(duty_roster.guarantor_duty.len(), 8);
@@ -191,16 +331,17 @@ mod tests {
 				assert_eq!(duty_roster.guarantor_duty.iter().filter(|&&j| j == Chain::Relay).count(), 2);
 			};
 
-			System::set_random_seed([0u8; 32].into());
+			system::Module::<Test>::set_random_seed([0u8; 32].into());
 			let duty_roster_0 = Parachains::calculate_duty_roster();
 			check_roster(&duty_roster_0);
 
-			System::set_random_seed([1u8; 32].into());
+			system::Module::<Test>::set_random_seed([1u8; 32].into());
 			let duty_roster_1 = Parachains::calculate_duty_roster();
 			check_roster(&duty_roster_1);
 			assert!(duty_roster_0 != duty_roster_1);
 
-			System::set_random_seed([2u8; 32].into());
+
+			system::Module::<Test>::set_random_seed([2u8; 32].into());
 			let duty_roster_2 = Parachains::calculate_duty_roster();
 			check_roster(&duty_roster_2);
 			assert!(duty_roster_0 != duty_roster_2);

polkadot/runtime/wasm/Cargo.lock

@@ -498,9 +498,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "substrate-codec"
 version = "0.1.0"
-dependencies = [
- "substrate-runtime-std 0.1.0",
-]
 
 [[package]]
 name = "substrate-keyring"

polkadot/service/src/lib.rs

@@ -351,7 +351,13 @@ impl Service {
 			// Load the first available key. Code above makes sure it exisis.
 			let key = keystore.load(&keystore.contents()?[0], "")?;
 			info!("Using authority key {:?}", key.public());
-			Some(consensus::Service::new(client.clone(), network.clone(), transaction_pool.clone(), key))
+			Some(consensus::Service::new(
+				client.clone(),
+				network.clone(),
+				transaction_pool.clone(),
+				::std::time::Duration::from_millis(4000), // TODO: dynamic
+				key,
+			))
 		} else {
 			None
 		};

polkadot/statement-table/src/generic.rs

@@ -268,6 +268,7 @@ pub struct Table<C: Context> {
 	authority_data: HashMap<C::AuthorityId, AuthorityData<C>>,
 	detected_misbehavior: HashMap<C::AuthorityId, <C as ResolveMisbehavior>::Misbehavior>,
 	candidate_votes: HashMap<C::Digest, CandidateData<C>>,
+	includable_count: HashMap<C::GroupId, usize>,
 }
 
 impl<C: Context> Default for Table<C> {
@@ -276,6 +277,7 @@ impl<C: Context> Default for Table<C> {
 			authority_data: HashMap::new(),
 			detected_misbehavior: HashMap::new(),
 			candidate_votes: HashMap::new(),
+			includable_count: HashMap::new(),
 		}
 	}
 }
@@ -294,6 +296,11 @@ impl<C: Context> Table<C> {
 		let mut best_candidates = BTreeMap::new();
 		for candidate_data in self.candidate_votes.values() {
 			let group_id = &candidate_data.group_id;
+
+			if !self.includable_count.contains_key(group_id) {
+				continue
+			}
+
 			let (validity_t, availability_t) = context.requisite_votes(group_id);
 
 			if !candidate_data.can_be_included(validity_t, availability_t) { continue }
@@ -394,6 +401,11 @@ impl<C: Context> Table<C> {
 		&self.detected_misbehavior
 	}
 
+	/// Get the current number of parachains with includable candidates.
+	pub fn includable_count(&self) -> usize {
+		self.includable_count.len()
+	}
+
 	/// Fill a statement batch and note messages as seen by the targets.
 	pub fn fill_batch<B>(&mut self, batch: &mut B)
 		where B: StatementBatch<
@@ -622,6 +634,9 @@ impl<C: Context> Table<C> {
 			Some(votes) => votes,
 		};
 
+		let (v_threshold, a_threshold) = context.requisite_votes(&votes.group_id);
+		let was_includable = votes.can_be_included(v_threshold, a_threshold);
+
 		// check that this authority actually can vote in this group.
 		if !context.is_member_of(&from, &votes.group_id) {
 			let (sig, valid) = match vote {
@@ -686,6 +701,9 @@ impl<C: Context> Table<C> {
 			}
 		}
 
+		let is_includable = votes.can_be_included(v_threshold, a_threshold);
+		update_includable_count(&mut self.includable_count, &votes.group_id, was_includable, is_includable);
+
 		(None, Some(votes.summary(digest)))
 	}
 
@@ -701,6 +719,9 @@ impl<C: Context> Table<C> {
 			Some(votes) => votes,
 		};
 
+		let (v_threshold, a_threshold) = context.requisite_votes(&votes.group_id);
+		let was_includable = votes.can_be_included(v_threshold, a_threshold);
+
 		// check that this authority actually can vote in this group.
 		if !context.is_availability_guarantor_of(&from, &votes.group_id) {
 			return (
@@ -716,10 +737,29 @@ impl<C: Context> Table<C> {
 		}
 
 		votes.availability_votes.insert(from, signature);
+
+		let is_includable = votes.can_be_included(v_threshold, a_threshold);
+		update_includable_count(&mut self.includable_count, &votes.group_id, was_includable, is_includable);
+
 		(None, Some(votes.summary(digest)))
 	}
 }
 
+fn update_includable_count<G: Hash + Eq + Clone>(map: &mut HashMap<G, usize>, group_id: &G, was_includable: bool, is_includable: bool) {
+	if was_includable && !is_includable {
+		if let Entry::Occupied(mut entry) = map.entry(group_id.clone()) {
+			*entry.get_mut() -= 1;
+			if *entry.get() == 0 {
+				entry.remove();
+			}
+		}
+	}
+
+	if !was_includable && is_includable {
+		*map.entry(group_id.clone()).or_insert(0) += 1;
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -746,11 +786,7 @@ mod tests {
 	}
 
 	fn create<C: Context>() -> Table<C> {
-		Table {
-			authority_data: HashMap::default(),
-			detected_misbehavior: HashMap::default(),
-			candidate_votes: HashMap::default(),
-		}
+		Table::default()
 	}
 
 	#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
@@ -806,8 +842,16 @@ mod tests {
 			self.authorities.get(authority).map(|v| &v.1 == group).unwrap_or(false)
 		}
 
-		fn requisite_votes(&self, _id: &GroupId) -> (usize, usize) {
-			(6, 34)
+		fn requisite_votes(&self, id: &GroupId) -> (usize, usize) {
+			let mut total_validity = 0;
+			let mut total_availability = 0;
+
+			for &(ref validity, ref availability) in self.authorities.values() {
+				if validity == id { total_validity += 1 }
+				if availability == id { total_availability += 1 }
+			}
+
+			(total_validity / 2 + 1, total_availability / 2 + 1)
 		}
 	}
 
@@ -1067,6 +1111,69 @@ mod tests {
 		assert!(!candidate.can_be_included(validity_threshold, availability_threshold));
 	}
 
+	#[test]
+	fn includability_counter() {
+		let context = TestContext {
+			authorities: {
+				let mut map = HashMap::new();
+				map.insert(AuthorityId(1), (GroupId(2), GroupId(455)));
+				map.insert(AuthorityId(2), (GroupId(2), GroupId(455)));
+				map.insert(AuthorityId(3), (GroupId(2), GroupId(455)));
+				map.insert(AuthorityId(4), (GroupId(455), GroupId(2)));
+				map
+			}
+		};
+
+		// have 2/3 validity guarantors note validity.
+		let mut table = create();
+		let statement = SignedStatement {
+			statement: Statement::Candidate(Candidate(2, 100)),
+			signature: Signature(1),
+			sender: AuthorityId(1),
+		};
+		let candidate_digest = Digest(100);
+
+		table.import_statement(&context, statement, None);
+		assert!(!table.detected_misbehavior.contains_key(&AuthorityId(1)));
+		assert!(!table.candidate_includable(&candidate_digest, &context));
+		assert!(table.includable_count.is_empty());
+
+		let vote = SignedStatement {
+			statement: Statement::Valid(candidate_digest.clone()),
+			signature: Signature(2),
+			sender: AuthorityId(2),
+		};
+
+		table.import_statement(&context, vote, None);
+		assert!(!table.detected_misbehavior.contains_key(&AuthorityId(2)));
+		assert!(!table.candidate_includable(&candidate_digest, &context));
+		assert!(table.includable_count.is_empty());
+
+		// have the availability guarantor note validity.
+		let vote = SignedStatement {
+			statement: Statement::Available(candidate_digest.clone()),
+			signature: Signature(4),
+			sender: AuthorityId(4),
+		};
+
+		table.import_statement(&context, vote, None);
+		assert!(!table.detected_misbehavior.contains_key(&AuthorityId(4)));
+		assert!(table.candidate_includable(&candidate_digest, &context));
+		assert!(table.includable_count.get(&GroupId(2)).is_some());
+
+		// have the last validity guarantor note invalidity. now it is unincludable.
+		let vote = SignedStatement {
+			statement: Statement::Invalid(candidate_digest.clone()),
+			signature: Signature(3),
+			sender: AuthorityId(3),
+		};
+
+		table.import_statement(&context, vote, None);
+		assert!(!table.detected_misbehavior.contains_key(&AuthorityId(2)));
+		assert!(!table.candidate_includable(&candidate_digest, &context));
+		assert!(table.includable_count.is_empty());
+	}
+
 	#[test]
 	fn candidate_import_gives_summary() {
 		let context = TestContext {

polkadot/transaction-pool/src/lib.rs

@@ -36,7 +36,8 @@ use std::sync::Arc;
 
 use polkadot_api::PolkadotApi;
 use primitives::{AccountId, Timestamp};
-use runtime::{Block, UncheckedExtrinsic, TimestampCall, Call};
+use primitives::parachain::CandidateReceipt;
+use runtime::{Block, UncheckedExtrinsic, TimestampCall, ParachainsCall, Call};
 use substrate_runtime_primitives::traits::{Bounded, Checkable};
 use transaction_pool::{Pool, Readiness};
 use transaction_pool::scoring::{Change, Choice};
@@ -59,17 +60,26 @@ pub struct PolkadotBlock {
 impl PolkadotBlock {
 	/// Create a new block, checking high-level well-formedness.
 	pub fn from(unchecked: Block) -> ::std::result::Result<Self, Block> {
-		if unchecked.extrinsics.len() < 1 {
+		if unchecked.extrinsics.len() < 2 {
 			return Err(unchecked);
 		}
 		if unchecked.extrinsics[0].is_signed() {
 			return Err(unchecked);
 		}
+		if unchecked.extrinsics[1].is_signed() {
+			return Err(unchecked);
+		}
+
 		match unchecked.extrinsics[0].extrinsic.function {
 			Call::Timestamp(TimestampCall::set(_)) => {},
 			_ => return Err(unchecked),
 		}
 
+		match unchecked.extrinsics[1].extrinsic.function {
+			Call::Parachains(ParachainsCall::set_heads(_)) => {},
+			_ => return Err(unchecked),
+		}
+
 		// any further checks...
 		Ok(PolkadotBlock { block: unchecked, location: None })
 	}
@@ -92,6 +102,19 @@ impl PolkadotBlock {
 			}
 		}
 	}
+
+	/// Retrieve the parachain candidates proposed for this block.
+	pub fn parachain_heads(&self) -> &[CandidateReceipt] {
+		if let Call::Parachains(ParachainsCall::set_heads(ref t)) = self.block.extrinsics[1].extrinsic.function {
+			&t[..]
+		} else {
+			if let Some((file, line)) = self.location {
+				panic!("Invalid block used in `PolkadotBlock::force_from` at {}:{}", file, line);
+			} else {
+				panic!("Invalid block made it through the PolkadotBlock verification!?");
+			}
+		}
+	}
 }
 
 #[macro_export]

polkadot/validator/Cargo.toml

@@ -1,11 +0,0 @@
-[package]
-name = "polkadot-validator"
-version = "0.1.0"
-authors = ["Parity Technologies <admin@parity.io>"]
-
-[dependencies]
-error-chain = "0.11"
-serde = "1.0"
-substrate-primitives = { path = "../../substrate/primitives" }
-substrate-serializer = { path = "../../substrate/serializer" }
-polkadot-primitives = { path = "../primitives" }

polkadot/validator/src/error.rs

@@ -1,33 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
-
-use serializer;
-
-error_chain! {
-	foreign_links {
-		Serialization(serializer::Error);
-	}
-	errors {
-		Timeout {
-			description("Validation task has timed-out."),
-			display("Validation timeout."),
-		}
-		InvalidCode(details: String) {
-			description("The code is invalid."),
-			display("invalid code: '{}'", details),
-		}
-	}
-}

polkadot/validator/src/lib.rs

@@ -1,34 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Validator implementation.
-
-#[warn(missing_docs)]
-
-extern crate substrate_primitives as primitives;
-extern crate substrate_serializer as serializer;
-extern crate polkadot_primitives;
-extern crate serde;
-
-#[macro_use]
-extern crate error_chain;
-
-mod error;
-mod parachains;
-mod validator;
-
-pub use error::{Error, ErrorKind, Result};
-pub use validator::Validator;

polkadot/validator/src/parachains.rs

@@ -1,68 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use std::fmt;
-
-use polkadot_primitives::validator;
-use serde::de::DeserializeOwned;
-
-use error::Result;
-
-/// Parachain code implementation.
-pub trait ParachainCode: fmt::Debug {
-	/// Deserialized message type.
-	type Message: DeserializeOwned;
-	/// Balance download.
-	type Download: DeserializeOwned;
-	/// Deserialized block data type.
-	type BlockData: DeserializeOwned;
-	/// Parachain head data.
-	type HeadData: DeserializeOwned;
-	/// Result
-	type Result: Into<validator::ValidationResult>;
-
-	/// Given decoded messages and proof validate it and return egress posts.
-	fn check(
-		&self,
-		messages: Vec<(u64, Vec<Self::Message>)>,
-		downloads: Vec<Self::Download>,
-		block_data: Self::BlockData,
-		head_data: Self::HeadData,
-	) -> Result<Self::Result>;
-}
-
-/// Dummy implementation of the first parachain validation.
-#[derive(Debug)]
-pub struct ParaChain1;
-
-impl ParachainCode for ParaChain1 {
-	type Message = ();
-	type Download = ();
-	type BlockData = ();
-	type HeadData = ();
-	type Result = validator::ValidationResult;
-
-	fn check(
-		&self,
-		_messages: Vec<(u64, Vec<Self::Message>)>,
-		_downloads: Vec<Self::Download>,
-		_block_data: Self::BlockData,
-		_head_data: Self::HeadData,
-	) -> Result<Self::Result>
-	{
-		unimplemented!()
-	}
-}

polkadot/validator/src/validator.rs

@@ -1,101 +0,0 @@
-// Copyright 2017 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-use std::fmt;
-
-use polkadot_primitives::{validator, parachain};
-use serde::de::DeserializeOwned;
-use serializer;
-
-use error::{ErrorKind, Result};
-use parachains::{ParachainCode, ParaChain1};
-
-/// A dummy validator implementation.
-#[derive(Debug)]
-pub struct Validator {
-	codes: Vec<Box<Code>>,
-}
-
-impl Validator {
-	/// Create a new validator.
-	pub fn new() -> Self {
-		Validator {
-			codes: vec![
-				Box::new(ParaChain1) as Box<Code>
-			],
-		}
-	}
-}
-
-impl Validator {
-	pub fn validate(
-		&self,
-		code: &[u8],
-		consolidated_ingress: &[(u64, Vec<parachain::Message>)],
-		balance_downloads: &[validator::BalanceDownload],
-		block_data: &parachain::BlockData,
-		previous_head_data: &parachain::HeadData,
-	) -> Result<validator::ValidationResult> {
-		ensure!(code.len() == 1, ErrorKind::InvalidCode(format!("The code should be a single byte.")));
-
-		match self.codes.get(code[0] as usize) {
-			Some(code) => code.check(consolidated_ingress, balance_downloads, block_data, previous_head_data),
-			None => bail!(ErrorKind::InvalidCode(format!("Unknown parachain code."))),
-		}
-	}
-}
-
-/// Simplified parachain code verification
-trait Code: fmt::Debug {
-	/// Given parachain candidate block data returns it's validity
-	/// and possible generated egress posts.
-	fn check(
-		&self,
-		consolidated_ingress: &[(u64, Vec<parachain::Message>)],
-		balance_downloads: &[validator::BalanceDownload],
-		block_data: &parachain::BlockData,
-		previous_head_data: &parachain::HeadData,
-	) -> Result<validator::ValidationResult>;
-}
-
-impl<M, B, T, R> Code for T where
-	M: DeserializeOwned,
-	B: DeserializeOwned,
-	R: Into<validator::ValidationResult>,
-	T: ParachainCode<Message=M, BlockData=B, Result=R>,
-{
-	fn check(
-		&self,
-		consolidated_ingress: &[(u64, Vec<parachain::Message>)],
-		balance_downloads: &[validator::BalanceDownload],
-		block_data: &parachain::BlockData,
-		previous_head_data: &parachain::HeadData,
-	) -> Result<validator::ValidationResult> {
-		let messages = consolidated_ingress.iter()
-			.map(|&(ref block, ref vec)| Ok((*block, vec.iter()
-				 .map(|msg| serializer::from_slice(&msg.0).map_err(Into::into))
-				 .collect::<Result<Vec<_>>>()?
-			)))
-			.collect::<Result<Vec<_>>>()?;
-		let downloads = balance_downloads.iter()
-			.map(|download| serializer::from_slice(&download.0).map_err(Into::into))
-			.collect::<Result<Vec<_>>>()?;
-		let block_data = serializer::from_slice(&block_data.0)?;
-		let head_data = serializer::from_slice(&previous_head_data.0)?;
-
-		Ok(self.check(messages, downloads, block_data, head_data)?.into())
-	}
-}