Collators get incoming parachain messages (#149)

* refactor out a consensus data fetcher from table router

* move statement checking logic into router

* refuse to start authority if collator

* support building the table router asynchronously

* instantiate_consensus does not overwrite old

* update key in new consensus if there was none before

* collator collects ingress from network

* test produced egress roots

* fix adder-collator compilation

* address first grumbles

* integrate new gossip with collator network launch

* address review

polkadot/Cargo.lock

@@ -24,7 +24,7 @@ version = "0.1.0"
 dependencies = [
  "adder 0.1.0",
  "ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "polkadot-collator 0.1.0",
@@ -670,11 +670,11 @@ dependencies = [
 
 [[package]]
 name = "exit-future"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
- "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2160,7 +2160,7 @@ dependencies = [
 name = "polkadot-cli"
 version = "0.3.0"
 dependencies = [
- "exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "polkadot-service 0.3.0",
@@ -2176,9 +2176,12 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-codec 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "polkadot-cli 0.3.0",
+ "polkadot-network 0.1.0",
  "polkadot-primitives 0.1.0",
  "polkadot-runtime 0.1.0",
+ "polkadot-validation 0.1.0",
  "substrate-client 0.1.0 (git+https://github.com/paritytech/substrate)",
+ "substrate-keyring 0.1.0 (git+https://github.com/paritytech/substrate)",
  "substrate-primitives 0.1.0 (git+https://github.com/paritytech/substrate)",
  "tokio 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -2208,6 +2211,7 @@ name = "polkadot-network"
 version = "0.1.0"
 dependencies = [
  "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-codec 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2348,7 +2352,7 @@ name = "polkadot-validation"
 version = "0.1.0"
 dependencies = [
  "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "parity-codec 3.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3540,7 +3544,7 @@ dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3929,7 +3933,7 @@ version = "0.3.0"
 source = "git+https://github.com/paritytech/substrate#45824913c980bb1ba3963f9bba67775a507d8624"
 dependencies = [
  "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4789,7 +4793,7 @@ dependencies = [
 "checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e"
 "checksum environmental 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db746025e3ea695bfa0ae744dbacd5fcfc8db51b9760cf8bd0ab69708bb93c49"
 "checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02"
-"checksum exit-future 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "87559b08e99a81a92bbb867d237543e43495857749f688e0773390a20d56c61c"
+"checksum exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d8013f441e38e31c670e7f34ec8f1d5d3a2bd9d303c1ff83976ca886005e8f48"
 "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
 "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
 "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"

polkadot/cli/src/lib.rs

@@ -45,6 +45,7 @@ pub use service::{
 
 pub use cli::{VersionInfo, IntoExit};
 pub use cli::error;
+pub use tokio::runtime::TaskExecutor;
 
 fn load_spec(id: &str) -> Result<Option<service::ChainSpec>, String> {
 	Ok(match ChainSpec::from(id) {
@@ -68,7 +69,7 @@ pub trait Worker: IntoExit {
 	fn configuration(&self) -> service::CustomConfiguration { Default::default() }
 
 	/// Do work and schedule exit.
-	fn work<S: PolkadotService>(self, service: &S) -> Self::Work;
+	fn work<S: PolkadotService>(self, service: &S, executor: TaskExecutor) -> Self::Work;
 }
 
 /// Parse command line arguments into service configuration.
@@ -129,7 +130,7 @@ fn run_until_exit<T, C, W>(
 	let executor = runtime.executor();
 	cli::informant::start(&service, exit.clone(), executor.clone());
 
-	let _ = runtime.block_on(worker.work(&*service));
+	let _ = runtime.block_on(worker.work(&*service, executor.clone()));
 	exit_send.fire();
 
 	// we eagerly drop the service so that the internal exit future is fired,

polkadot/collator/Cargo.toml

@@ -12,5 +12,10 @@ substrate-primitives = { git = "https://github.com/paritytech/substrate" }
 polkadot-runtime = { path = "../runtime", version = "0.1" }
 polkadot-primitives = { path = "../primitives", version = "0.1" }
 polkadot-cli = { path = "../cli" }
+polkadot-network = { path = "../network" }
+polkadot-validation = { path = "../validation" }
 log = "0.4"
 tokio = "0.1.7"
+
+[dev-dependencies]
+substrate-keyring = { git = "https://github.com/paritytech/substrate" }

polkadot/collator/src/lib.rs

@@ -53,25 +53,33 @@ extern crate tokio;
 extern crate polkadot_cli;
 extern crate polkadot_runtime;
 extern crate polkadot_primitives;
+extern crate polkadot_network;
+extern crate polkadot_validation;
 
 #[macro_use]
 extern crate log;
 
-use std::collections::{BTreeSet, BTreeMap, HashSet};
+#[cfg(test)]
+extern crate substrate_keyring as keyring;
+
+use std::collections::HashSet;
 use std::fmt;
 use std::sync::Arc;
 use std::time::Duration;
 
-use futures::{future, stream, Stream, Future, IntoFuture};
+use futures::{future, Stream, Future, IntoFuture};
 use client::BlockchainEvents;
 use primitives::{ed25519, Pair};
-use polkadot_primitives::{BlockId, SessionKey};
+use polkadot_primitives::{BlockId, SessionKey, Hash, Block};
-use polkadot_primitives::parachain::{self, BlockData, DutyRoster, HeadData, ConsolidatedIngress, Message, Id as ParaId};
+use polkadot_primitives::parachain::{self, BlockData, DutyRoster, HeadData, ConsolidatedIngress, Message, Id as ParaId, Extrinsic};
 use polkadot_cli::{PolkadotService, CustomConfiguration, CoreApi, ParachainHost};
-use polkadot_cli::{Worker, IntoExit, ProvideRuntimeApi};
+use polkadot_cli::{Worker, IntoExit, ProvideRuntimeApi, TaskExecutor};
+use polkadot_network::validation::{ValidationNetwork, SessionParams};
+use polkadot_network::NetworkService;
 use tokio::timer::Timeout;
 
 pub use polkadot_cli::VersionInfo;
+pub use polkadot_network::validation::Incoming;
 
 const COLLATION_TIMEOUT: Duration = Duration::from_secs(30);
 
@@ -107,63 +115,23 @@ pub trait ParachainContext: Clone {
 		&self,
 		last_head: HeadData,
 		ingress: I,
-	) -> Result<(BlockData, HeadData), InvalidHead>;
+	) -> Result<(BlockData, HeadData, Extrinsic), InvalidHead>;
 }
 
 /// Relay chain context needed to collate.
 /// This encapsulates a network and local database which may store
 /// some of the input.
 pub trait RelayChainContext {
-	type Error;
+	type Error: ::std::fmt::Debug;
 
 	/// Future that resolves to the un-routed egress queues of a parachain.
 	/// The first item is the oldest.
-	type FutureEgress: IntoFuture<Item=Vec<Vec<Message>>, Error=Self::Error>;
+	type FutureEgress: IntoFuture<Item=ConsolidatedIngress, Error=Self::Error>;
-
-	/// Provide a set of all parachains meant to be routed to at a block.
-	fn routing_parachains(&self) -> BTreeSet<ParaId>;
 
 	/// Get un-routed egress queues from a parachain to the local parachain.
 	fn unrouted_egress(&self, id: ParaId) -> Self::FutureEgress;
 }
 
-/// Collate the necessary ingress queue using the given context.
-pub fn collate_ingress<'a, R>(relay_context: R)
-	-> impl Future<Item=ConsolidatedIngress, Error=R::Error> + 'a
-	where
-		R: RelayChainContext,
-		R::Error: 'a,
-		R::FutureEgress: 'a,
-{
-	let mut egress_fetch = Vec::new();
-
-	for routing_parachain in relay_context.routing_parachains() {
-		let fetch = relay_context
-			.unrouted_egress(routing_parachain)
-			.into_future()
-			.map(move |egresses| (routing_parachain, egresses));
-
-		egress_fetch.push(fetch);
-	}
-
-	// create a map ordered first by the depth of the egress queue
-	// and then by the parachain ID.
-	//
-	// then transform that into the consolidated egress queue.
-	stream::futures_unordered(egress_fetch)
-		.fold(BTreeMap::new(), |mut map, (routing_id, egresses)| {
-			for (depth, egress) in egresses.into_iter().rev().enumerate() {
-				let depth = -(depth as i64);
-				map.insert((depth, routing_id), egress);
-			}
-
-			Ok(map)
-		})
-		.map(|ordered| ordered.into_iter().map(|((_, id), egress)| (id, egress)))
-		.map(|i| i.collect::<Vec<_>>())
-		.map(ConsolidatedIngress)
-}
-
 /// Produce a candidate for the parachain, with given contexts, parent head, and signing key.
 pub fn collate<'a, R, P>(
 	local_id: ParaId,
@@ -174,19 +142,22 @@ pub fn collate<'a, R, P>(
 )
 	-> impl Future<Item=parachain::Collation, Error=Error<R::Error>> + 'a
 	where
-		R: RelayChainContext + 'a,
+		R: RelayChainContext,
 		R::Error: 'a,
 		R::FutureEgress: 'a,
 		P: ParachainContext + 'a,
 {
-	collate_ingress(relay_context).map_err(Error::Polkadot).and_then(move |ingress| {
+	let ingress = relay_context.unrouted_egress(local_id).into_future().map_err(Error::Polkadot);
-		let (block_data, head_data) = para_context.produce_candidate(
+	ingress.and_then(move |ConsolidatedIngress(ingress)| {
+		let (block_data, head_data, mut extrinsic) = para_context.produce_candidate(
 			last_head,
-			ingress.0.iter().flat_map(|&(id, ref msgs)| msgs.iter().cloned().map(move |msg| (id, msg)))
+			ingress.iter().flat_map(|&(id, ref msgs)| msgs.iter().cloned().map(move |msg| (id, msg)))
 		).map_err(Error::Collator)?;
 
 		let block_data_hash = block_data.hash();
 		let signature = key.sign(block_data_hash.as_ref()).into();
+		let egress_queue_roots =
+			::polkadot_validation::egress_roots(&mut extrinsic.outgoing_messages);
 
 		let receipt = parachain::CandidateReceipt {
 			parachain_index: local_id,
@@ -194,11 +165,12 @@ pub fn collate<'a, R, P>(
 			signature,
 			head_data,
 			balance_uploads: Vec::new(),
-			egress_queue_roots: Vec::new(),
+			egress_queue_roots,
 			fees: 0,
 			block_data_hash,
 		};
 
+		// not necessary to send extrinsic because it is recomputed from execution.
 		Ok(parachain::Collation {
 			receipt,
 			block_data,
@@ -207,18 +179,34 @@ pub fn collate<'a, R, P>(
 }
 
 /// Polkadot-api context.
-struct ApiContext;
+struct ApiContext<P, E> {
-
+	network: ValidationNetwork<P, E, NetworkService, TaskExecutor>,
-impl RelayChainContext for ApiContext {
+	parent_hash: Hash,
-	type Error = client::error::Error;
+	authorities: Vec<SessionKey>,
-	type FutureEgress = Result<Vec<Vec<Message>>, Self::Error>;
-
-	fn routing_parachains(&self) -> BTreeSet<ParaId> {
-		BTreeSet::new()
 }
 
-	fn unrouted_egress(&self, _id: ParaId) -> Self::FutureEgress {
+impl<P: 'static, E: 'static> RelayChainContext for ApiContext<P, E> where
-		Ok(Vec::new())
+	P: ProvideRuntimeApi + Send + Sync,
+	P::Api: ParachainHost<Block>,
+	E: Future<Item=(),Error=()> + Clone + Send + Sync + 'static,
+{
+	type Error = String;
+	type FutureEgress = Box<Future<Item=ConsolidatedIngress, Error=String> + Send>;
+
+	fn unrouted_egress(&self, id: ParaId) -> Self::FutureEgress {
+		let session = self.network.instantiate_session(SessionParams {
+			local_session_key: None,
+			parent_hash: self.parent_hash,
+			authorities: self.authorities.clone(),
+		}).map_err(|e| format!("unable to instantiate validation session: {:?}", e));
+
+		let fetch_incoming = session
+			.and_then(move |session| session.fetch_incoming(id).map_err(|e|
+				format!("unable to fetch incoming data: {:?}", e)
+			))
+			.map(ConsolidatedIngress);
+
+		Box::new(fetch_incoming)
 	}
 }
 
@@ -241,7 +229,7 @@ impl<P, E> IntoExit for CollationNode<P, E> where
 
 impl<P, E> Worker for CollationNode<P, E> where
 	P: ParachainContext + Send + 'static,
-	E: Future<Item=(),Error=()> + Clone + Send + 'static
+	E: Future<Item=(),Error=()> + Clone + Send + Sync + 'static
 {
 	type Work = Box<Future<Item=(),Error=()> + Send>;
 
@@ -254,13 +242,42 @@ impl<P, E> Worker for CollationNode<P, E> where
 		config
 	}
 
-	fn work<S>(self, service: &S) -> Self::Work
+	fn work<S>(self, service: &S, task_executor: TaskExecutor) -> Self::Work
 		where S: PolkadotService,
 	{
-
 		let CollationNode { parachain_context, exit, para_id, key } = self;
 		let client = service.client();
 		let network = service.network();
+		let known_oracle = client.clone();
+
+		let message_validator = polkadot_network::gossip::register_validator(
+			&*network,
+			move |block_hash: &Hash| {
+				use client::{BlockStatus, ChainHead};
+				use polkadot_network::gossip::Known;
+
+				match known_oracle.block_status(&BlockId::hash(*block_hash)) {
+					Err(_) | Ok(BlockStatus::Unknown) | Ok(BlockStatus::Queued) => None,
+					Ok(BlockStatus::KnownBad) => Some(Known::Bad),
+					Ok(BlockStatus::InChain) => match known_oracle.leaves() {
+						Err(_) => None,
+						Ok(leaves) => if leaves.contains(block_hash) {
+							Some(Known::Leaf)
+						} else {
+							Some(Known::Old)
+						},
+					}
+				}
+			},
+		);
+
+		let validation_network = ValidationNetwork::new(
+			network.clone(),
+			exit.clone(),
+			message_validator,
+			client.clone(),
+			task_executor,
+		);
 
 		let inner_exit = exit.clone();
 		let work = client.import_notification_stream()
@@ -269,7 +286,9 @@ impl<P, E> Worker for CollationNode<P, E> where
 					($e:expr) => {
 						match $e {
 							Ok(x) => x,
-							Err(e) => return future::Either::A(future::err(Error::Polkadot(e))),
+							Err(e) => return future::Either::A(future::err(Error::Polkadot(
+								format!("{:?}", e)
+							))),
 						}
 					}
 				}
@@ -281,6 +300,7 @@ impl<P, E> Worker for CollationNode<P, E> where
 				let client = client.clone();
 				let key = key.clone();
 				let parachain_context = parachain_context.clone();
+				let validation_network = validation_network.clone();
 
 				let work = future::lazy(move || {
 					let api = client.runtime_api();
@@ -289,16 +309,24 @@ impl<P, E> Worker for CollationNode<P, E> where
 						None => return future::Either::A(future::ok(())),
 					};
 
+					let authorities = try_fr!(api.authorities(&id));
+
 					let targets = compute_targets(
 						para_id,
-						try_fr!(api.authorities(&id)).as_slice(),
+						authorities.as_slice(),
 						try_fr!(api.duty_roster(&id)),
 					);
 
+					let context = ApiContext {
+						network: validation_network,
+						parent_hash: relay_parent,
+						authorities,
+					};
+
 					let collation_work = collate(
 						para_id,
 						HeadData(last_head),
-						ApiContext,
+						context,
 						parachain_context,
 						key,
 					).map(move |collation| {
@@ -355,7 +383,7 @@ pub fn run_collator<P, E, I, ArgT>(
 ) -> polkadot_cli::error::Result<()> where
 	P: ParachainContext + Send + 'static,
 	E: IntoFuture<Item=(),Error=()>,
-	E::Future: Send + Clone + 'static,
+	E::Future: Send + Clone + Sync + 'static,
 	I: IntoIterator<Item=ArgT>,
 	ArgT: Into<std::ffi::OsString> + Clone,
 {
@@ -365,74 +393,91 @@ pub fn run_collator<P, E, I, ArgT>(
 
 #[cfg(test)]
 mod tests {
+	use std::collections::HashMap;
+	use polkadot_primitives::parachain::OutgoingMessage;
+	use keyring::AuthorityKeyring;
 	use super::*;
 
-	use std::collections::{HashMap, BTreeSet};
+	#[derive(Default, Clone)]
-
+	struct DummyRelayChainContext {
-	use futures::Future;
+		ingress: HashMap<ParaId, ConsolidatedIngress>
-	use polkadot_primitives::parachain::{Message, Id as ParaId};
-
-	pub struct DummyRelayChainCtx {
-		egresses: HashMap<ParaId, Vec<Vec<Message>>>,
-		currently_routing: BTreeSet<ParaId>,
 	}
 
-	impl RelayChainContext for DummyRelayChainCtx {
+	impl RelayChainContext for DummyRelayChainContext {
 		type Error = ();
-		type FutureEgress = Result<Vec<Vec<Message>>, ()>;
+		type FutureEgress = Box<Future<Item=ConsolidatedIngress,Error=()>>;
 
-		fn routing_parachains(&self) -> BTreeSet<ParaId> {
+		fn unrouted_egress(&self, para_id: ParaId) -> Self::FutureEgress {
-			self.currently_routing.clone()
+			match self.ingress.get(&para_id) {
+				Some(ingress) => Box::new(future::ok(ingress.clone())),
+				None => Box::new(future::empty()),
+			}
+		}
 	}
 
-		fn unrouted_egress(&self, id: ParaId) -> Result<Vec<Vec<Message>>, ()> {
+	#[derive(Clone)]
-			Ok(self.egresses.get(&id).cloned().unwrap_or_default())
+	struct DummyParachainContext;
+
+	impl ParachainContext for DummyParachainContext {
+		fn produce_candidate<I: IntoIterator<Item=(ParaId, Message)>>(
+			&self,
+			_last_head: HeadData,
+			ingress: I,
+		) -> Result<(BlockData, HeadData, Extrinsic), InvalidHead> {
+			// send messages right back.
+			Ok((
+				BlockData(vec![1, 2, 3, 4, 5,]),
+				HeadData(vec![9, 9, 9]),
+				Extrinsic {
+					outgoing_messages: ingress.into_iter().map(|(id, msg)| OutgoingMessage {
+						target: id,
+						data: msg.0,
+					}).collect(),
+				}
+			))
 		}
 	}
 
 	#[test]
-	fn collates_ingress() {
+	fn collates_correct_queue_roots() {
-		let route_from = |x: &[ParaId]| {
+		let mut context = DummyRelayChainContext::default();
-			 let mut set = BTreeSet::new();
-			 set.extend(x.iter().cloned());
-			 set
-		};
 
-		let message = |x: Vec<u8>| vec![Message(x)];
+		let id = ParaId::from(100);
 
-		let dummy_ctx = DummyRelayChainCtx {
+		let a = ParaId::from(123);
-			currently_routing: route_from(&[2.into(), 3.into()]),
+		let b = ParaId::from(456);
-			egresses: vec![
-				// egresses for `2`: last routed successfully 5 blocks ago.
-				(2.into(), vec![
-					message(vec![1, 2, 3]),
-					message(vec![4, 5, 6]),
-					message(vec![7, 8]),
-					message(vec![10]),
-					message(vec![12]),
-				]),
 
-				// egresses for `3`: last routed successfully 3 blocks ago.
+		let messages_from_a = vec![
-				(3.into(), vec![
+			Message(vec![1, 1, 1]),
-					message(vec![9]),
+			Message(b"helloworld".to_vec()),
-					message(vec![11]),
+		];
-					message(vec![13]),
+		let messages_from_b = vec![
-				]),
+			Message(b"dogglesworth".to_vec()),
-			].into_iter().collect(),
+			Message(b"buy_1_chili_con_carne_here_is_my_cash".to_vec()),
-		};
+		];
 
-		assert_eq!(
+		let root_a = ::polkadot_validation::message_queue_root(
-			collate_ingress(dummy_ctx).wait().unwrap(),
+			messages_from_a.iter().map(|msg| &msg.0)
-			ConsolidatedIngress(vec![
+		);
-				(2.into(), message(vec![1, 2, 3])),
+
-				(2.into(), message(vec![4, 5, 6])),
+		let root_b = ::polkadot_validation::message_queue_root(
-				(2.into(), message(vec![7, 8])),
+			messages_from_b.iter().map(|msg| &msg.0)
-				(3.into(), message(vec![9])),
+		);
-				(2.into(), message(vec![10])),
+
-				(3.into(), message(vec![11])),
+		context.ingress.insert(id, ConsolidatedIngress(vec![
-				(2.into(), message(vec![12])),
+			(b, messages_from_b),
-				(3.into(), message(vec![13])),
+			(a, messages_from_a),
-			]
+		]));
-		))
+
+		let collation = collate(
+			id,
+			HeadData(vec![5]),
+			context.clone(),
+			DummyParachainContext,
+			AuthorityKeyring::Alice.pair().into(),
+		).wait().unwrap();
+
+		// ascending order by root.
+		assert_eq!(collation.receipt.egress_queue_roots, vec![(a, root_a), (b, root_b)]);
 	}
 }

polkadot/network/Cargo.toml

@@ -19,6 +19,7 @@ futures = "0.1"
 tokio = "0.1.7"
 log = "0.4"
 slice-group-by = "0.2.2"
+exit-future = "0.1.4"
 
 [dev-dependencies]
 substrate-client = { git = "https://github.com/paritytech/substrate" }

polkadot/network/src/lib.rs

@@ -32,6 +32,7 @@ extern crate arrayvec;
 extern crate parking_lot;
 extern crate tokio;
 extern crate slice_group_by;
+extern crate exit_future;
 
 #[macro_use]
 extern crate futures;
@@ -66,7 +67,6 @@ use self::local_collations::LocalCollations;
 
 use std::collections::{HashMap, HashSet};
 
-
 #[cfg(test)]
 mod tests;
 
@@ -213,10 +213,13 @@ impl PolkadotProtocol {
 	fn new_validation_session(
 		&mut self,
 		ctx: &mut Context<Block>,
-		parent_hash: Hash,
+		params: validation::SessionParams,
-		session: validation::ValidationSession,
+	) -> validation::ValidationSession {
-	) {
+
-		if let Some(new_local) = self.live_validation_sessions.new_validation_session(parent_hash, session) {
+		let (session, new_local) = self.live_validation_sessions
+			.new_validation_session(params);
+
+		if let Some(new_local) = new_local {
 			for (id, peer_data) in self.peers.iter_mut()
 				.filter(|&(_, ref info)| info.should_send_key())
 			{
@@ -227,10 +230,13 @@ impl PolkadotProtocol {
 				));
 			}
 		}
+
+		session
 	}
 
-	fn remove_validation_session(&mut self, parent_hash: &Hash) {
+	// true indicates that it was removed actually.
-		self.live_validation_sessions.remove(parent_hash);
+	fn remove_validation_session(&mut self, parent_hash: Hash) -> bool {
+		self.live_validation_sessions.remove(parent_hash)
 	}
 
 	fn dispatch_pending_requests(&mut self, ctx: &mut Context<Block>) {

polkadot/network/src/router.rs

@@ -14,10 +14,11 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Statement routing and consensus table router implementation.
+//! Statement routing and validation statement table router implementation.
 //!
-//! During the consensus process, validators exchange statements on validity and availability
+//! During the attestation process, validators exchange statements on validity and availability
 //! of parachain candidates.
+//!
 //! The `Router` in this file hooks into the underlying network to fulfill
 //! the `TableRouter` trait from `polkadot-validation`, which is expected to call into a shared statement table
 //! and dispatch evaluation work as necessary when new statements come in.
@@ -33,18 +34,15 @@ use polkadot_primitives::parachain::{
 };
 
 use codec::{Encode, Decode};
-use futures::{future, prelude::*};
+use futures::prelude::*;
-use futures::sync::oneshot::{self, Receiver};
 use parking_lot::Mutex;
 
-use std::collections::{hash_map::{Entry, HashMap}, HashSet};
+use std::collections::{HashMap, HashSet};
-use std::{io, mem};
+use std::io;
 use std::sync::Arc;
 
-use gossip::{RegisteredMessageValidator};
+use validation::{self, SessionDataFetcher, NetworkService, Executor};
-use validation::{NetworkService, Knowledge, Executor};
 
-type IngressPair = (ParaId, Vec<Message>);
 type IngressPairRef<'a> = (ParaId, &'a [Message]);
 
 /// Compute the gossip topic for attestations on the given parent hash.
@@ -55,109 +53,48 @@ pub(crate) fn attestation_topic(parent_hash: Hash) -> Hash {
 	BlakeTwo256::hash(&v[..])
 }
 
-fn incoming_message_topic(parent_hash: Hash, parachain: ParaId) -> Hash {
-	let mut v = parent_hash.as_ref().to_vec();
-	parachain.using_encoded(|s| v.extend(s));
-	v.extend(b"incoming");
-
-	BlakeTwo256::hash(&v[..])
-}
-
-/// Receiver for block data.
-pub struct BlockDataReceiver {
-	outer: Receiver<Receiver<BlockData>>,
-	inner: Option<Receiver<BlockData>>
-}
-
-impl Future for BlockDataReceiver {
-	type Item = BlockData;
-	type Error = io::Error;
-
-	fn poll(&mut self) -> Poll<BlockData, io::Error> {
-		let map_err = |_| io::Error::new(
-			io::ErrorKind::Other,
-			"Sending end of channel hung up",
-		);
-
-		if let Some(ref mut inner) = self.inner {
-			return inner.poll().map_err(map_err);
-		}
-		match self.outer.poll().map_err(map_err)? {
-			Async::Ready(mut inner) => {
-				let poll_result = inner.poll();
-				self.inner = Some(inner);
-				poll_result.map_err(map_err)
-			}
-			Async::NotReady => Ok(Async::NotReady),
-		}
-	}
-}
-/// receiver for incoming data.
-#[derive(Clone)]
-pub struct IncomingReceiver {
-	inner: future::Shared<Receiver<Incoming>>
-}
-
-impl Future for IncomingReceiver {
-	type Item = Incoming;
-	type Error = io::Error;
-
-	fn poll(&mut self) -> Poll<Incoming, io::Error> {
-		match self.inner.poll() {
-			Ok(Async::NotReady) => Ok(Async::NotReady),
-			Ok(Async::Ready(i)) => Ok(Async::Ready(Incoming::clone(&*i))),
-			Err(_) => Err(io::Error::new(
-				io::ErrorKind::Other,
-				"Sending end of channel hung up",
-			)),
-		}
-	}
-}
-
 /// Table routing implementation.
 pub struct Router<P, E, N: NetworkService, T> {
 	table: Arc<SharedTable>,
-	network: Arc<N>,
-	api: Arc<P>,
-	exit: E,
-	task_executor: T,
-	parent_hash: Hash,
 	attestation_topic: Hash,
-	knowledge: Arc<Mutex<Knowledge>>,
+	fetcher: SessionDataFetcher<P, E, N, T>,
-	fetch_incoming: Arc<Mutex<HashMap<ParaId, IncomingReceiver>>>,
 	deferred_statements: Arc<Mutex<DeferredStatements>>,
-	message_validator: RegisteredMessageValidator,
 }
 
 impl<P, E, N: NetworkService, T> Router<P, E, N, T> {
 	pub(crate) fn new(
 		table: Arc<SharedTable>,
-		network: Arc<N>,
+		fetcher: SessionDataFetcher<P, E, N, T>,
-		api: Arc<P>,
-		task_executor: T,
-		parent_hash: Hash,
-		knowledge: Arc<Mutex<Knowledge>>,
-		exit: E,
-		message_validator: RegisteredMessageValidator,
 	) -> Self {
+		let parent_hash = fetcher.parent_hash();
 		Router {
 			table,
-			network,
-			api,
-			task_executor,
-			parent_hash,
 			attestation_topic: attestation_topic(parent_hash),
-			knowledge,
-			fetch_incoming: Arc::new(Mutex::new(HashMap::new())),
 			deferred_statements: Arc::new(Mutex::new(DeferredStatements::new())),
-			exit,
+			fetcher,
-			message_validator,
 		}
 	}
 
-	/// Get the attestation topic for gossip.
+	/// Return a future of checked messages. These should be imported into the router
-	pub(crate) fn gossip_topic(&self) -> Hash {
+	/// with `import_statement`.
-		self.attestation_topic
+	pub(crate) fn checked_statements(&self) -> impl Stream<Item=SignedStatement,Error=()> {
+		// spin up a task in the background that processes all incoming statements
+		// validation has been done already by the gossip validator.
+		// this will block internally until the gossip messages stream is obtained.
+		self.network().gossip_messages_for(self.attestation_topic)
+			.filter_map(|msg| {
+				debug!(target: "validation", "Processing statement for live validation session");
+				crate::gossip::GossipMessage::decode(&mut &msg[..])
+			})
+			.map(|msg| msg.statement)
+	}
+
+	fn parent_hash(&self) -> Hash {
+		self.fetcher.parent_hash()
+	}
+
+	fn network(&self) -> &Arc<N> {
+		self.fetcher.network()
 	}
 }
 
@@ -165,16 +102,9 @@ impl<P, E: Clone, N: NetworkService, T: Clone> Clone for Router<P, E, N, T> {
 	fn clone(&self) -> Self {
 		Router {
 			table: self.table.clone(),
-			network: self.network.clone(),
+			fetcher: self.fetcher.clone(),
-			api: self.api.clone(),
-			task_executor: self.task_executor.clone(),
-			parent_hash: self.parent_hash.clone(),
 			attestation_topic: self.attestation_topic.clone(),
 			deferred_statements: self.deferred_statements.clone(),
-			fetch_incoming: self.fetch_incoming.clone(),
-			knowledge: self.knowledge.clone(),
-			exit: self.exit.clone(),
-			message_validator: self.message_validator.clone(),
 		}
 	}
 }
@@ -187,7 +117,7 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 {
 	/// Import a statement whose signature has been checked already.
 	pub(crate) fn import_statement(&self, statement: SignedStatement) {
-		trace!(target: "p_net", "importing consensus statement {:?}", statement.statement);
+		trace!(target: "p_net", "importing validation statement {:?}", statement.statement);
 
 		// defer any statements for which we haven't imported the candidate yet
 		let c_hash = {
@@ -214,7 +144,7 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		};
 
 		// prepend the candidate statement.
-		debug!(target: "consensus", "Importing statements about candidate {:?}", c_hash);
+		debug!(target: "validation", "Importing statements about candidate {:?}", c_hash);
 		statements.insert(0, statement);
 		let producers: Vec<_> = self.table.import_remote_statements(
 			self,
@@ -222,12 +152,12 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		);
 		// dispatch future work as necessary.
 		for (producer, statement) in producers.into_iter().zip(statements) {
-			self.knowledge.lock().note_statement(statement.sender, &statement.statement);
+			self.fetcher.knowledge().lock().note_statement(statement.sender, &statement.statement);
 
 			if let Some(work) = producer.map(|p| self.create_work(c_hash, p)) {
-				trace!(target: "consensus", "driving statement work to completion");
+				trace!(target: "validation", "driving statement work to completion");
-				let work = work.select2(self.exit.clone()).then(|_| Ok(()));
+				let work = work.select2(self.fetcher.exit().clone()).then(|_| Ok(()));
-				self.task_executor.spawn(work);
+				self.fetcher.executor().spawn(work);
 			}
 		}
 	}
@@ -251,14 +181,15 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 				group_messages.clear(); // reuse allocation from previous iterations.
 				group_messages.extend(group.iter().map(|msg| msg.data.clone()).map(Message));
 
-				debug!(target: "consensus", "Circulating messages from {:?} to {:?} at {}",
+				debug!(target: "valdidation", "Circulating messages from {:?} to {:?} at {}",
-					source, target, self.parent_hash);
+					source, target, self.parent_hash());
 
 				// this is the ingress from source to target, with given messages.
-				let target_incoming = incoming_message_topic(self.parent_hash, target);
+				let target_incoming =
+					validation::incoming_message_topic(self.parent_hash(), target);
 				let ingress_for: IngressPairRef = (source, &group_messages[..]);
 
-				self.network.gossip_message(target_incoming, ingress_for.encode());
+				self.network().gossip_message(target_incoming, ingress_for.encode());
 			}
 		}
 	}
@@ -269,11 +200,11 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		D: Future<Item=(BlockData, Incoming),Error=io::Error> + Send + 'static,
 	{
 		let table = self.table.clone();
-		let network = self.network.clone();
+		let network = self.network().clone();
-		let knowledge = self.knowledge.clone();
+		let knowledge = self.fetcher.knowledge().clone();
 		let attestation_topic = self.attestation_topic.clone();
 
-		producer.prime(self.api.clone())
+		producer.prime(self.fetcher.api().clone())
 			.map(move |validated| {
 				// store the data before broadcasting statements, so other peers can fetch.
 				knowledge.lock().note_candidate(
@@ -289,61 +220,6 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 			})
 			.map_err(|e| debug!(target: "p_net", "Failed to produce statements: {:?}", e))
 	}
-
-}
-
-impl<P: ProvideRuntimeApi, E, N, T> Router<P, E, N, T> where
-	P::Api: ParachainHost<Block>,
-	N: NetworkService,
-	T: Executor,
-	E: Future<Item=(),Error=()> + Clone + Send + 'static,
-{
-	fn do_fetch_incoming(&self, parachain: ParaId) -> IncomingReceiver {
-		use polkadot_primitives::BlockId;
-		let (tx, rx) = {
-			let mut fetching = self.fetch_incoming.lock();
-			match fetching.entry(parachain) {
-				Entry::Occupied(entry) => return entry.get().clone(),
-				Entry::Vacant(entry) => {
-					// has not been requested yet.
-					let (tx, rx) = oneshot::channel();
-					let rx = IncomingReceiver { inner: rx.shared() };
-					entry.insert(rx.clone());
-
-					(tx, rx)
-				}
-			}
-		};
-
-		let parent_hash = self.parent_hash;
-		let topic = incoming_message_topic(parent_hash, parachain);
-		let gossip_messages = self.network.gossip_messages_for(topic)
-			.map_err(|()| panic!("unbounded receivers do not throw errors; qed"))
-			.filter_map(|msg| IngressPair::decode(&mut msg.as_slice()));
-
-		let canon_roots = self.api.runtime_api().ingress(&BlockId::hash(parent_hash), parachain)
-			.map_err(|e| format!("Cannot fetch ingress for parachain {:?} at {:?}: {:?}",
-				parachain, parent_hash, e)
-			);
-
-		let work = canon_roots.into_future()
-			.and_then(move |ingress_roots| match ingress_roots {
-				None => Err(format!("No parachain {:?} registered at {}", parachain, parent_hash)),
-				Some(roots) => Ok(roots.into_iter().collect())
-			})
-			.and_then(move |ingress_roots| ComputeIngress {
-				inner: gossip_messages,
-				ingress_roots,
-				incoming: Vec::new(),
-			})
-			.map(move |incoming| if let Some(i) = incoming { let _ = tx.send(i); })
-			.select2(self.exit.clone())
-			.then(|_| Ok(()));
-
-		self.task_executor.spawn(work);
-
-		rx
-	}
 }
 
 impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> where
@@ -353,8 +229,8 @@ impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> wh
 	E: Future<Item=(),Error=()> + Clone + Send + 'static,
 {
 	type Error = io::Error;
-	type FetchCandidate = BlockDataReceiver;
+	type FetchCandidate = validation::BlockDataReceiver;
-	type FetchIncoming = IncomingReceiver;
+	type FetchIncoming = validation::IncomingReceiver;
 
 	fn local_candidate(&self, receipt: CandidateReceipt, block_data: BlockData, extrinsic: Extrinsic) {
 		// produce a signed statement
@@ -363,42 +239,22 @@ impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> wh
 		let statement = self.table.import_validated(validated);
 
 		// give to network to make available.
-		self.knowledge.lock().note_candidate(hash, Some(block_data), Some(extrinsic));
+		self.fetcher.knowledge().lock().note_candidate(hash, Some(block_data), Some(extrinsic));
-		self.network.gossip_message(self.attestation_topic, statement.encode());
+		self.network().gossip_message(self.attestation_topic, statement.encode());
 	}
 
-	fn fetch_block_data(&self, candidate: &CandidateReceipt) -> BlockDataReceiver {
+	fn fetch_block_data(&self, candidate: &CandidateReceipt) -> Self::FetchCandidate {
-		let parent_hash = self.parent_hash.clone();
+		self.fetcher.fetch_block_data(candidate)
-		let candidate = candidate.clone();
-		let (tx, rx) = ::futures::sync::oneshot::channel();
-		self.network.with_spec(move |spec, ctx| {
-			let inner_rx = spec.fetch_block_data(ctx, &candidate, parent_hash);
-			let _ = tx.send(inner_rx);
-		});
-		BlockDataReceiver { outer: rx, inner: None }
 	}
 
 	fn fetch_incoming(&self, parachain: ParaId) -> Self::FetchIncoming {
-		self.do_fetch_incoming(parachain)
+		self.fetcher.fetch_incoming(parachain)
 	}
 }
 
 impl<P, E, N: NetworkService, T> Drop for Router<P, E, N, T> {
 	fn drop(&mut self) {
-		let parent_hash = self.parent_hash.clone();
+		self.fetcher.network().drop_gossip(self.attestation_topic);
-		self.network.with_spec(move |spec, _| spec.remove_validation_session(&parent_hash));
-		self.network.drop_gossip(self.attestation_topic);
-
-		{
-			let mut incoming_fetched = self.fetch_incoming.lock();
-			for (para_id, _) in incoming_fetched.drain() {
-				self.network.drop_gossip(incoming_message_topic(
-					self.parent_hash,
-					para_id,
-				));
-			}
-		}
-		self.message_validator.remove_session(&parent_hash);
 	}
 }
 
@@ -457,63 +313,10 @@ impl DeferredStatements {
 	}
 }
 
-// computes ingress from incoming stream of messages.
-// returns `None` if the stream concludes too early.
-#[must_use = "futures do nothing unless polled"]
-struct ComputeIngress<S> {
-	ingress_roots: HashMap<ParaId, Hash>,
-	incoming: Vec<IngressPair>,
-	inner: S,
-}
-
-impl<S> Future for ComputeIngress<S> where S: Stream<Item=IngressPair> {
-	type Item = Option<Incoming>;
-	type Error = S::Error;
-
-	fn poll(&mut self) -> Poll<Option<Incoming>, Self::Error> {
-		loop {
-			if self.ingress_roots.is_empty() {
-				return Ok(Async::Ready(
-					Some(mem::replace(&mut self.incoming, Vec::new()))
-				))
-			}
-
-			let (para_id, messages) = match try_ready!(self.inner.poll()) {
-				None => return Ok(Async::Ready(None)),
-				Some(next) => next,
-			};
-
-			match self.ingress_roots.entry(para_id) {
-				Entry::Vacant(_) => continue,
-				Entry::Occupied(occupied) => {
-					let canon_root = occupied.get().clone();
-					let messages = messages.iter().map(|m| &m.0[..]);
-					if ::polkadot_validation::message_queue_root(messages) != canon_root {
-						continue;
-					}
-
-					occupied.remove();
-				}
-			}
-
-			let pos = self.incoming.binary_search_by_key(
-				&para_id,
-				|&(id, _)| id,
-			)
-				.err()
-				.expect("incoming starts empty and only inserted when \
-					para_id not inserted before; qed");
-
-			self.incoming.insert(pos, (para_id, messages));
-		}
-	}
-}
-
 #[cfg(test)]
 mod tests {
 	use super::*;
 	use substrate_primitives::crypto::UncheckedInto;
-	use futures::stream;
 	use polkadot_primitives::parachain::ValidatorId;
 
 	#[test]
@@ -556,70 +359,4 @@ mod tests {
 			assert!(traces.is_empty());
 		}
 	}
-
-	#[test]
-	fn compute_ingress_works() {
-		let actual_messages = [
-			(
-				ParaId::from(1),
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-			(
-				ParaId::from(2),
-				vec![
-					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
-					Message(b"hello world".to_vec()),
-				],
-			),
-			(
-				ParaId::from(5),
-				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
-			),
-		];
-
-		let roots: HashMap<_, _> = actual_messages.iter()
-			.map(|&(para_id, ref messages)| (
-				para_id,
-				::polkadot_validation::message_queue_root(messages.iter().map(|m| &m.0)),
-			))
-			.collect();
-
-		let inputs = [
-			(
-				ParaId::from(1), // wrong message.
-				vec![Message(vec![1, 1, 2, 2]), Message(vec![3, 3, 4, 4])],
-			),
-			(
-				ParaId::from(1),
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-			(
-				ParaId::from(1), // duplicate
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-
-			(
-				ParaId::from(5), // out of order
-				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
-			),
-			(
-				ParaId::from(1234), // un-routed parachain.
-				vec![Message(vec![9, 9, 9, 9])],
-			),
-			(
-				ParaId::from(2),
-				vec![
-					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
-					Message(b"hello world".to_vec()),
-				],
-			),
-		];
-		let ingress = ComputeIngress {
-			ingress_roots: roots,
-			incoming: Vec::new(),
-			inner: stream::iter_ok::<_, ()>(inputs.iter().cloned()),
-		};
-
-		assert_eq!(ingress.wait().unwrap().unwrap(), actual_messages);
-	}
 }

polkadot/network/src/tests/mod.rs

@@ -17,11 +17,10 @@
 //! Tests for polkadot and validation network.
 
 use super::{PolkadotProtocol, Status, Message, FullStatus};
-use validation::{ValidationSession, Knowledge};
+use validation::SessionParams;
 
-use parking_lot::Mutex;
 use polkadot_validation::GenericStatement;
-use polkadot_primitives::{Block, SessionKey};
+use polkadot_primitives::{Block, Hash, SessionKey};
 use polkadot_primitives::parachain::{CandidateReceipt, HeadData, BlockData, CollatorId, ValidatorId};
 use substrate_primitives::crypto::UncheckedInto;
 use codec::Encode;
@@ -31,7 +30,6 @@ use substrate_network::{
 	generic_message::Message as GenericMessage
 };
 
-use std::sync::Arc;
 use futures::Future;
 
 mod validation;
@@ -88,11 +86,12 @@ fn make_status(status: &Status, roles: Roles) -> FullStatus {
 	}
 }
 
-fn make_validation_session(local_key: SessionKey) -> (ValidationSession, Arc<Mutex<Knowledge>>) {
+fn make_validation_session(parent_hash: Hash, local_key: SessionKey) -> SessionParams {
-	let knowledge = Arc::new(Mutex::new(Knowledge::new()));
+	SessionParams {
-	let c = ValidationSession::new(knowledge.clone(), local_key);
+		local_session_key: Some(local_key),
-
+		parent_hash,
-	(c, knowledge)
+		authorities: Vec::new(),
+	}
 }
 
 fn on_message(protocol: &mut PolkadotProtocol, ctx: &mut TestContext, from: NodeIndex, message: Message) {
@@ -120,8 +119,8 @@ fn sends_session_key() {
 
 	{
 		let mut ctx = TestContext::default();
-		let (session, _knowledge) = make_validation_session(local_key.clone());
+		let params = make_validation_session(parent_hash, local_key.clone());
-		protocol.new_validation_session(&mut ctx, parent_hash, session);
+		protocol.new_validation_session(&mut ctx, params);
 		assert!(ctx.has_message(peer_a, Message::SessionKey(local_key.clone())));
 	}
 
@@ -160,8 +159,9 @@ fn fetches_from_those_with_knowledge() {
 
 	let status = Status { collating_for: None };
 
-	let (session, knowledge) = make_validation_session(local_key.clone());
+	let params = make_validation_session(parent_hash, local_key.clone());
-	protocol.new_validation_session(&mut TestContext::default(), parent_hash, session);
+	let session = protocol.new_validation_session(&mut TestContext::default(), params);
+	let knowledge = session.knowledge();
 
 	knowledge.lock().note_statement(a_key.clone(), &GenericStatement::Valid(candidate_hash));
 	let recv = protocol.fetch_block_data(&mut TestContext::default(), &candidate_receipt, parent_hash);
@@ -290,11 +290,11 @@ fn many_session_keys() {
 	let local_key_a: ValidatorId = [3; 32].unchecked_into();
 	let local_key_b: ValidatorId = [4; 32].unchecked_into();
 
-	let (session_a, _knowledge_a) = make_validation_session(local_key_a.clone());
+	let params_a = make_validation_session(parent_a, local_key_a.clone());
-	let (session_b, _knowledge_b) = make_validation_session(local_key_b.clone());
+	let params_b = make_validation_session(parent_b, local_key_b.clone());
 
-	protocol.new_validation_session(&mut TestContext::default(), parent_a, session_a);
+	protocol.new_validation_session(&mut TestContext::default(), params_a);
-	protocol.new_validation_session(&mut TestContext::default(), parent_b, session_b);
+	protocol.new_validation_session(&mut TestContext::default(), params_b);
 
 	assert_eq!(protocol.live_validation_sessions.recent_keys(), &[local_key_a.clone(), local_key_b.clone()]);
 
@@ -313,7 +313,7 @@ fn many_session_keys() {
 
 	let peer_b = 2;
 
-	protocol.remove_validation_session(&parent_a);
+	assert!(protocol.remove_validation_session(parent_a));
 
 	{
 		let mut ctx = TestContext::default();

polkadot/network/src/tests/validation.rs

@@ -471,9 +471,12 @@ fn ingress_fetch_works() {
 	};
 
 	// make sure everyone can get ingress for their own parachain.
-	let fetch_a = router_a.fetch_incoming(id_a).map_err(|_| format!("Could not fetch ingress_a"));
+	let fetch_a = router_a.then(move |r| r.unwrap()
-	let fetch_b = router_b.fetch_incoming(id_b).map_err(|_| format!("Could not fetch ingress_b"));
+		.fetch_incoming(id_a).map_err(|_| format!("Could not fetch ingress_a")));
-	let fetch_c = router_c.fetch_incoming(id_c).map_err(|_| format!("Could not fetch ingress_c"));
+	let fetch_b = router_b.then(move |r| r.unwrap()
+		.fetch_incoming(id_b).map_err(|_| format!("Could not fetch ingress_b")));
+	let fetch_c = router_c.then(move |r| r.unwrap()
+		.fetch_incoming(id_c).map_err(|_| format!("Could not fetch ingress_c")));
 
 	let work = fetch_a.join3(fetch_b, fetch_c);
 	runtime.spawn(built.gossip.then(|_| Ok(()))); // in background.

polkadot/network/src/validation.rs

@@ -19,19 +19,23 @@
 //! This fulfills the `polkadot_validation::Network` trait, providing a hook to be called
 //! each time a validation session begins on a new chain head.
 
-use sr_primitives::traits::ProvideRuntimeApi;
+use sr_primitives::traits::{BlakeTwo256, ProvideRuntimeApi, Hash as HashT};
 use substrate_network::Context as NetContext;
 use polkadot_validation::{Network as ParachainNetwork, SharedTable, Collators, Statement, GenericStatement};
-use polkadot_primitives::{Block, Hash};
+use polkadot_primitives::{Block, Hash, SessionKey};
-use polkadot_primitives::parachain::{Id as ParaId, Collation, Extrinsic, ParachainHost, BlockData, ValidatorId,
+use polkadot_primitives::parachain::{
-	CollatorId};
+	Id as ParaId, Collation, Extrinsic, ParachainHost, BlockData, Message, CandidateReceipt,
-use codec::Decode;
+	CollatorId, ValidatorId,
+};
+use codec::{Encode, Decode};
 
 use futures::prelude::*;
-use futures::future::Executor as FutureExecutor;
+use futures::future::{self, Executor as FutureExecutor};
 use futures::sync::mpsc;
+use futures::sync::oneshot::{self, Receiver};
 
-use std::collections::HashMap;
+use std::collections::hash_map::{HashMap, Entry};
+use std::io;
 use std::sync::Arc;
 
 use arrayvec::ArrayVec;
@@ -39,10 +43,12 @@ use tokio::runtime::TaskExecutor;
 use parking_lot::Mutex;
 
 use router::Router;
-use gossip::{POLKADOT_ENGINE_ID, GossipMessage, RegisteredMessageValidator, MessageValidationData};
+use gossip::{POLKADOT_ENGINE_ID, RegisteredMessageValidator, MessageValidationData};
 
 use super::PolkadotProtocol;
 
+pub use polkadot_validation::Incoming;
+
 /// An executor suitable for dispatching async consensus tasks.
 pub trait Executor {
 	fn spawn<F: Future<Item=(),Error=()> + Send + 'static>(&self, f: F);
@@ -111,54 +117,14 @@ impl NetworkService for super::NetworkService {
 	}
 }
 
-// task that processes all gossipped consensus messages,
+/// Params to a current validation session.
-// checking signatures
+pub struct SessionParams {
-struct MessageProcessTask<P, E, N: NetworkService, T> {
+	/// The local session key.
-	inner_stream: mpsc::UnboundedReceiver<Vec<u8>>,
+	pub local_session_key: Option<SessionKey>,
-	table_router: Router<P, E, N, T>,
+	/// The parent hash.
-}
+	pub parent_hash: Hash,
-
+	/// The authorities.
-impl<P, E, N, T> MessageProcessTask<P, E, N, T> where
+	pub authorities: Vec<SessionKey>,
-	P: ProvideRuntimeApi + Send + Sync + 'static,
-	P::Api: ParachainHost<Block>,
-	E: Future<Item=(),Error=()> + Clone + Send + 'static,
-	N: NetworkService,
-	T: Clone + Executor + Send + 'static,
-{
-	fn process_message(&self, msg: Vec<u8>) -> Option<Async<()>> {
-		debug!(target: "validation", "Processing consensus statement for live consensus");
-
-		// statements are already checked by gossip validator.
-		if let Some(message) = GossipMessage::decode(&mut &msg[..]) {
-			self.table_router.import_statement(message.statement);
-		}
-
-		None
-	}
-}
-
-impl<P, E, N, T> Future for MessageProcessTask<P, E, N, T> where
-	P: ProvideRuntimeApi + Send + Sync + 'static,
-	P::Api: ParachainHost<Block>,
-	E: Future<Item=(),Error=()> + Clone + Send + 'static,
-	N: NetworkService,
-	T: Clone + Executor + Send + 'static,
-{
-	type Item = ();
-	type Error = ();
-
-	fn poll(&mut self) -> Poll<(), ()> {
-		loop {
-			match self.inner_stream.poll() {
-				Ok(Async::Ready(Some(val))) => if let Some(async) = self.process_message(val) {
-					return Ok(async);
-				},
-				Ok(Async::Ready(None)) => return Ok(Async::Ready(())),
-				Ok(Async::NotReady) => return Ok(Async::NotReady),
-				Err(e) => debug!(target: "p_net", "Error getting consensus message: {:?}", e),
-			}
-		}
-	}
 }
 
 /// Wrapper around the network service
@@ -195,70 +161,107 @@ impl<P, E: Clone, N, T: Clone> Clone for ValidationNetwork<P, E, N, T> {
 	}
 }
 
+impl<P, E, N, T> ValidationNetwork<P, E, N, T> where
+	P: ProvideRuntimeApi + Send + Sync + 'static,
+	P::Api: ParachainHost<Block>,
+	E: Clone + Future<Item=(),Error=()> + Send + Sync + 'static,
+	N: NetworkService,
+	T: Clone + Executor + Send + Sync + 'static,
+{
+	/// Instantiate session data fetcher at a parent hash.
+	///
+	/// If the used session key is new, it will be broadcast to peers.
+	/// If a validation session was already instantiated at this parent hash,
+	/// the underlying instance will be shared.
+	///
+	/// If there was already a validation session instantiated and a different
+	/// session key was set, then the new key will be ignored.
+	///
+	/// This implies that there can be multiple services intantiating validation
+	/// session instances safely, but they should all be coordinated on which session keys
+	/// are being used.
+	pub fn instantiate_session(&self, params: SessionParams)
+		-> oneshot::Receiver<SessionDataFetcher<P, E, N, T>>
+	{
+		let parent_hash = params.parent_hash;
+		let network = self.network.clone();
+		let api = self.api.clone();
+		let task_executor = self.executor.clone();
+		let exit = self.exit.clone();
+		let message_validator = self.message_validator.clone();
+
+		let (tx, rx) = oneshot::channel();
+		self.network.with_spec(move |spec, ctx| {
+			// before requesting messages, note live consensus session.
+			message_validator.note_session(
+				parent_hash,
+				MessageValidationData { authorities: params.authorities.clone() },
+			);
+
+			let session = spec.new_validation_session(ctx, params);
+			let _ = tx.send(SessionDataFetcher {
+				network,
+				api,
+				task_executor,
+				parent_hash,
+				knowledge: session.knowledge().clone(),
+				exit,
+				fetch_incoming: session.fetched_incoming().clone(),
+				message_validator,
+			});
+		});
+
+		rx
+	}
+}
+
 /// A long-lived network which can create parachain statement  routing processes on demand.
 impl<P, E, N, T> ParachainNetwork for ValidationNetwork<P, E, N, T> where
 	P: ProvideRuntimeApi + Send + Sync + 'static,
 	P::Api: ParachainHost<Block>,
-	E: Clone + Future<Item=(),Error=()> + Send + 'static,
+	E: Clone + Future<Item=(),Error=()> + Send + Sync + 'static,
 	N: NetworkService,
-	T: Clone + Executor + Send + 'static,
+	T: Clone + Executor + Send + Sync + 'static,
 {
+	type Error = String;
 	type TableRouter = Router<P, E, N, T>;
+	type BuildTableRouter = Box<Future<Item=Self::TableRouter,Error=String> + Send>;
 
 	fn communication_for(
 		&self,
 		table: Arc<SharedTable>,
 		outgoing: polkadot_validation::Outgoing,
 		authorities: &[ValidatorId],
-	) -> Self::TableRouter {
+	) -> Self::BuildTableRouter {
 		let parent_hash = table.consensus_parent_hash().clone();
-
-		let knowledge = Arc::new(Mutex::new(Knowledge::new()));
-
 		let local_session_key = table.session_key();
+
+		let build_fetcher = self.instantiate_session(SessionParams {
+			local_session_key: Some(local_session_key),
+			parent_hash,
+			authorities: authorities.to_vec(),
+		});
+
+		let executor = self.executor.clone();
+		let work = build_fetcher
+			.map_err(|e| format!("{:?}", e))
+			.map(move |fetcher| {
 				let table_router = Router::new(
 					table,
-			self.network.clone(),
+					fetcher,
-			self.api.clone(),
-			self.executor.clone(),
-			parent_hash,
-			knowledge.clone(),
-			self.exit.clone(),
-			self.message_validator.clone(),
 				);
 
 				table_router.broadcast_egress(outgoing);
 
-		let attestation_topic = table_router.gossip_topic();
-
 				let table_router_clone = table_router.clone();
-		let executor = self.executor.clone();
+				let work = table_router.checked_statements()
-		let exit = self.exit.clone();
+					.for_each(move |msg| { table_router_clone.import_statement(msg); Ok(()) });
-
+				executor.spawn(work);
-		// before requesting messages, note live consensus session.
-		self.message_validator.note_session(
-			parent_hash,
-			MessageValidationData { authorities: authorities.to_vec() },
-		);
-
-		// spin up a task in the background that processes all incoming statements
-		// TODO: propagate statements on a timer?
-		let inner_stream = self.network.gossip_messages_for(attestation_topic);
-		self.network
-			.with_spec(move |spec, ctx| {
-				spec.new_validation_session(ctx, parent_hash, ValidationSession {
-					knowledge,
-					local_session_key,
-				});
-				let process_task = MessageProcessTask {
-					inner_stream,
-					table_router: table_router_clone,
-				};
-
-				executor.spawn(process_task.select(exit).then(|_| Ok(())));
-		});
 
 				table_router
+			});
+
+		Box::new(work)
 	}
 }
 
@@ -368,21 +371,69 @@ impl Knowledge {
 	}
 }
 
+/// receiver for incoming data.
+#[derive(Clone)]
+pub struct IncomingReceiver {
+	inner: future::Shared<Receiver<Incoming>>
+}
+
+impl Future for IncomingReceiver {
+	type Item = Incoming;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Incoming, io::Error> {
+		match self.inner.poll() {
+			Ok(Async::NotReady) => Ok(Async::NotReady),
+			Ok(Async::Ready(i)) => Ok(Async::Ready(Incoming::clone(&*i))),
+			Err(_) => Err(io::Error::new(
+				io::ErrorKind::Other,
+				"Sending end of channel hung up",
+			)),
+		}
+	}
+}
+
+/// Incoming message gossip topic for a parachain at a given block hash.
+pub(crate) fn incoming_message_topic(parent_hash: Hash, parachain: ParaId) -> Hash {
+	let mut v = parent_hash.as_ref().to_vec();
+	parachain.using_encoded(|s| v.extend(s));
+	v.extend(b"incoming");
+
+	BlakeTwo256::hash(&v[..])
+}
+
 /// A current validation session instance.
+#[derive(Clone)]
 pub(crate) struct ValidationSession {
+	parent_hash: Hash,
 	knowledge: Arc<Mutex<Knowledge>>,
-	local_session_key: ValidatorId,
+	local_session_key: Option<ValidatorId>,
+	fetch_incoming: Arc<Mutex<FetchIncoming>>,
 }
 
 impl ValidationSession {
-	#[cfg(test)]
+	/// Create a new validation session instance. Needs to be attached to the
-	pub(crate) fn new(knowledge: Arc<Mutex<Knowledge>>, local_session_key: ValidatorId) -> Self {
+	/// nework.
+	pub(crate) fn new(params: SessionParams) -> Self {
 		ValidationSession {
-			knowledge,
+			parent_hash: params.parent_hash,
-			local_session_key
+			knowledge: Arc::new(Mutex::new(Knowledge::new())),
+			local_session_key: params.local_session_key,
+			fetch_incoming: Arc::new(Mutex::new(FetchIncoming::new())),
 		}
 	}
 
+	/// Get a handle to the shared knowledge relative to this consensus
+	/// instance.
+	pub(crate) fn knowledge(&self) -> &Arc<Mutex<Knowledge>> {
+		&self.knowledge
+	}
+
+	/// Get a handle to the shared list of parachains' incoming data fetch.
+	pub(crate) fn fetched_incoming(&self) -> &Arc<Mutex<FetchIncoming>> {
+		&self.fetch_incoming
+	}
+
 	// execute a closure with locally stored block data for a candidate, or a slice of session identities
 	// we believe should have the data.
 	fn with_block_data<F, U>(&self, hash: &Hash, f: F) -> U
@@ -447,8 +498,8 @@ impl RecentValidatorIds {
 pub(crate) struct LiveValidationSessions {
 	// recent local session keys.
 	recent: RecentValidatorIds,
-	// live validation session instances, on `parent_hash`.
+	// live validation session instances, on `parent_hash`. refcount retained alongside.
-	live_instances: HashMap<Hash, ValidationSession>,
+	live_instances: HashMap<Hash, (usize, ValidationSession)>,
 }
 
 impl LiveValidationSessions {
@@ -462,33 +513,74 @@ impl LiveValidationSessions {
 
 	/// Note new validation session. If the used session key is new,
 	/// it returns it to be broadcasted to peers.
+	///
+	/// If there was already a validation session instantiated and a different
+	/// session key was set, then the new key will be ignored.
 	pub(crate) fn new_validation_session(
 		&mut self,
-		parent_hash: Hash,
+		params: SessionParams,
-		session: ValidationSession,
+	) -> (ValidationSession, Option<ValidatorId>) {
-	) -> Option<ValidatorId> {
+		let parent_hash = params.parent_hash.clone();
-		let inserted_key = self.recent.insert(session.local_session_key.clone());
+
-		let maybe_new = if let InsertedRecentKey::New(_) = inserted_key {
+		let key = params.local_session_key.clone();
-			Some(session.local_session_key.clone())
+		let recent = &mut self.recent;
+
+		let mut check_new_key = || {
+			let inserted_key = key.clone().map(|key| recent.insert(key));
+			if let Some(InsertedRecentKey::New(_)) = inserted_key {
+				key.clone()
+			} else {
+				None
+			}
+		};
+
+		if let Some(&mut (ref mut rc, ref mut prev)) = self.live_instances.get_mut(&parent_hash) {
+			let maybe_new = if prev.local_session_key.is_none() {
+				prev.local_session_key = key.clone();
+				check_new_key()
 			} else {
 				None
 			};
 
-		self.live_instances.insert(parent_hash, session);
+			*rc += 1;
-
+			return (prev.clone(), maybe_new)
-		maybe_new
 		}
 
-	/// Remove validation session.
+		let session = ValidationSession::new(params);
-	pub(crate) fn remove(&mut self, parent_hash: &Hash) {
+		self.live_instances.insert(parent_hash, (1, session.clone()));
-		if let Some(validation_session) = self.live_instances.remove(parent_hash) {
+
+		(session, check_new_key())
+	}
+
+	/// Remove validation session. true indicates that it was actually removed.
+	pub(crate) fn remove(&mut self, parent_hash: Hash) -> bool {
+		let maybe_removed = if let Entry::Occupied(mut entry) = self.live_instances.entry(parent_hash) {
+			entry.get_mut().0 -= 1;
+			if entry.get().0 == 0 {
+				let (_, session) = entry.remove();
+				Some(session)
+			} else {
+				None
+			}
+		} else {
+			None
+		};
+
+		let session = match maybe_removed {
+			None => return false,
+			Some(s) => s,
+		};
+
+		if let Some(ref key) = session.local_session_key {
 			let key_still_used = self.live_instances.values()
-				.any(|c| c.local_session_key == validation_session.local_session_key);
+				.any(|c| c.1.local_session_key.as_ref() == Some(key));
 
 			if !key_still_used {
-				self.recent.remove(&validation_session.local_session_key)
+				self.recent.remove(key)
 			}
 		}
+
+		true
 	}
 
 	/// Recent session keys as a slice.
@@ -505,15 +597,290 @@ impl LiveValidationSessions {
 		where F: FnOnce(Result<&BlockData, Option<&[ValidatorId]>>) -> U
 	{
 		match self.live_instances.get(parent_hash) {
-			Some(c) => c.with_block_data(c_hash, |res| f(res.map_err(Some))),
+			Some(c) => c.1.with_block_data(c_hash, |res| f(res.map_err(Some))),
 			None => f(Err(None))
 		}
 	}
 }
 
+/// Receiver for block data.
+pub struct BlockDataReceiver {
+	outer: Receiver<Receiver<BlockData>>,
+	inner: Option<Receiver<BlockData>>
+}
+
+impl Future for BlockDataReceiver {
+	type Item = BlockData;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<BlockData, io::Error> {
+		let map_err = |_| io::Error::new(
+			io::ErrorKind::Other,
+			"Sending end of channel hung up",
+		);
+
+		if let Some(ref mut inner) = self.inner {
+			return inner.poll().map_err(map_err);
+		}
+		match self.outer.poll().map_err(map_err)? {
+			Async::Ready(inner) => {
+				self.inner = Some(inner);
+				self.poll()
+			}
+			Async::NotReady => Ok(Async::NotReady),
+		}
+	}
+}
+
+/// Wrapper around bookkeeping for tracking which parachains we're fetching incoming messages
+/// for.
+pub(crate) struct FetchIncoming {
+	exit_signal: ::exit_future::Signal,
+	parachains_fetching: HashMap<ParaId, IncomingReceiver>,
+}
+
+impl FetchIncoming {
+	fn new() -> Self {
+		FetchIncoming {
+			exit_signal: ::exit_future::signal_only(),
+			parachains_fetching: HashMap::new(),
+		}
+	}
+
+	// registers intent to fetch incoming. returns an optional piece of work
+	// that, if some, is needed to be run to completion in order for the future to
+	// resolve.
+	//
+	// impl Future has a bug here where it wrongly assigns a `'static` bound to `M`.
+	fn fetch_with_work<M, W>(&mut self, para_id: ParaId, make_work: M)
+		-> (IncomingReceiver, Option<Box<Future<Item=(),Error=()> + Send>>) where
+		M: FnOnce() -> W,
+		W: Future<Item=Option<Incoming>> + Send + 'static,
+	{
+		let (tx, rx) = match self.parachains_fetching.entry(para_id) {
+			Entry::Occupied(entry) => return (entry.get().clone(), None),
+			Entry::Vacant(entry) => {
+				// has not been requested yet.
+				let (tx, rx) = oneshot::channel();
+				let rx = IncomingReceiver { inner: rx.shared() };
+				entry.insert(rx.clone());
+
+				(tx, rx)
+			}
+		};
+
+		let exit = self.exit_signal.make_exit();
+		let work = make_work()
+			.map(move |incoming| if let Some(i) = incoming { let _ = tx.send(i); })
+			.select2(exit)
+			.then(|_| Ok(()));
+
+		(rx, Some(Box::new(work)))
+	}
+}
+
+/// Can fetch data for a given validation session
+pub struct SessionDataFetcher<P, E, N: NetworkService, T> {
+	network: Arc<N>,
+	api: Arc<P>,
+	fetch_incoming: Arc<Mutex<FetchIncoming>>,
+	exit: E,
+	task_executor: T,
+	knowledge: Arc<Mutex<Knowledge>>,
+	parent_hash: Hash,
+	message_validator: RegisteredMessageValidator,
+}
+
+impl<P, E, N: NetworkService, T> SessionDataFetcher<P, E, N, T> {
+	/// Get the parent hash.
+	pub(crate) fn parent_hash(&self) -> Hash {
+		self.parent_hash.clone()
+	}
+
+	/// Get the shared knowledge.
+	pub(crate) fn knowledge(&self) -> &Arc<Mutex<Knowledge>> {
+		&self.knowledge
+	}
+
+	/// Get the exit future.
+	pub(crate) fn exit(&self) -> &E {
+		&self.exit
+	}
+
+	/// Get the network service.
+	pub(crate) fn network(&self) -> &Arc<N> {
+		&self.network
+	}
+
+	/// Get the executor.
+	pub(crate) fn executor(&self) -> &T {
+		&self.task_executor
+	}
+
+	/// Get the runtime API.
+	pub(crate) fn api(&self) -> &Arc<P> {
+		&self.api
+	}
+}
+
+impl<P, E: Clone, N: NetworkService, T: Clone> Clone for SessionDataFetcher<P, E, N, T> {
+	fn clone(&self) -> Self {
+		SessionDataFetcher {
+			network: self.network.clone(),
+			api: self.api.clone(),
+			task_executor: self.task_executor.clone(),
+			parent_hash: self.parent_hash.clone(),
+			fetch_incoming: self.fetch_incoming.clone(),
+			knowledge: self.knowledge.clone(),
+			exit: self.exit.clone(),
+			message_validator: self.message_validator.clone(),
+		}
+	}
+}
+
+impl<P: ProvideRuntimeApi + Send, E, N, T> SessionDataFetcher<P, E, N, T> where
+	P::Api: ParachainHost<Block>,
+	N: NetworkService,
+	T: Clone + Executor + Send + 'static,
+	E: Future<Item=(),Error=()> + Clone + Send + 'static,
+{
+	/// Fetch block data for the given candidate receipt.
+	pub fn fetch_block_data(&self, candidate: &CandidateReceipt) -> BlockDataReceiver {
+		let parent_hash = self.parent_hash;
+		let candidate = candidate.clone();
+		let (tx, rx) = ::futures::sync::oneshot::channel();
+		self.network.with_spec(move |spec, ctx| {
+			let inner_rx = spec.fetch_block_data(ctx, &candidate, parent_hash);
+			let _ = tx.send(inner_rx);
+		});
+		BlockDataReceiver { outer: rx, inner: None }
+	}
+
+	/// Fetch incoming messages for a parachain.
+	pub fn fetch_incoming(&self, parachain: ParaId) -> IncomingReceiver {
+		use polkadot_primitives::BlockId;
+
+		let (rx, work) = self.fetch_incoming.lock().fetch_with_work(parachain.clone(), move || {
+			let parent_hash: Hash = self.parent_hash();
+			let topic = incoming_message_topic(parent_hash, parachain);
+
+			let gossip_messages = self.network().gossip_messages_for(topic)
+				.map_err(|()| panic!("unbounded receivers do not throw errors; qed"))
+				.filter_map(|msg| IngressPair::decode(&mut msg.as_slice()));
+
+			let canon_roots = self.api.runtime_api().ingress(&BlockId::hash(parent_hash), parachain)
+				.map_err(|e| format!("Cannot fetch ingress for parachain {:?} at {:?}: {:?}",
+					parachain, parent_hash, e)
+				);
+
+			canon_roots.into_future()
+				.and_then(move |ingress_roots| match ingress_roots {
+					None => Err(format!("No parachain {:?} registered at {}", parachain, parent_hash)),
+					Some(roots) => Ok(roots.into_iter().collect())
+				})
+				.and_then(move |ingress_roots| ComputeIngress {
+					inner: gossip_messages,
+					ingress_roots,
+					incoming: Vec::new(),
+				})
+				.select2(self.exit.clone())
+				.map(|res| match res {
+					future::Either::A((incoming, _)) => incoming,
+					future::Either::B(_) => None,
+				})
+		});
+
+		if let Some(work) = work {
+			self.task_executor.spawn(work);
+		}
+
+		rx
+	}
+}
+
+impl<P, E, N: NetworkService, T> Drop for SessionDataFetcher<P, E, N, T> {
+	fn drop(&mut self) {
+		// a bit of a hack...
+		let network = self.network.clone();
+		let fetch_incoming = self.fetch_incoming.clone();
+		let message_validator = self.message_validator.clone();
+
+		let parent_hash = self.parent_hash();
+
+		self.network.with_spec(move |spec, _| {
+			if !spec.remove_validation_session(parent_hash) { return }
+
+			let mut incoming_fetched = fetch_incoming.lock();
+			for (para_id, _) in incoming_fetched.parachains_fetching.drain() {
+				network.drop_gossip(incoming_message_topic(
+					parent_hash,
+					para_id,
+				));
+			}
+
+			message_validator.remove_session(&parent_hash);
+		});
+	}
+}
+
+type IngressPair = (ParaId, Vec<Message>);
+
+// computes ingress from incoming stream of messages.
+// returns `None` if the stream concludes too early.
+#[must_use = "futures do nothing unless polled"]
+struct ComputeIngress<S> {
+	ingress_roots: HashMap<ParaId, Hash>,
+	incoming: Vec<IngressPair>,
+	inner: S,
+}
+
+impl<S> Future for ComputeIngress<S> where S: Stream<Item=IngressPair> {
+	type Item = Option<Incoming>;
+	type Error = S::Error;
+
+	fn poll(&mut self) -> Poll<Option<Incoming>, Self::Error> {
+		loop {
+			if self.ingress_roots.is_empty() {
+				return Ok(Async::Ready(
+					Some(::std::mem::replace(&mut self.incoming, Vec::new()))
+				))
+			}
+
+			let (para_id, messages) = match try_ready!(self.inner.poll()) {
+				None => return Ok(Async::Ready(None)),
+				Some(next) => next,
+			};
+
+			match self.ingress_roots.entry(para_id) {
+				Entry::Vacant(_) => continue,
+				Entry::Occupied(occupied) => {
+					let canon_root = occupied.get().clone();
+					let messages = messages.iter().map(|m| &m.0[..]);
+					if ::polkadot_validation::message_queue_root(messages) != canon_root {
+						continue;
+					}
+
+					occupied.remove();
+				}
+			}
+
+			let pos = self.incoming.binary_search_by_key(
+				&para_id,
+				|&(id, _)| id,
+			)
+				.err()
+				.expect("incoming starts empty and only inserted when \
+					para_id not inserted before; qed");
+
+			self.incoming.insert(pos, (para_id, messages));
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
+	use futures::stream;
 	use substrate_primitives::crypto::UncheckedInto;
 
 	#[test]
@@ -565,4 +932,107 @@ mod tests {
 			_ => panic!("not new"),
 		}
 	}
+
+	#[test]
+	fn compute_ingress_works() {
+		let actual_messages = [
+			(
+				ParaId::from(1),
+				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
+			),
+			(
+				ParaId::from(2),
+				vec![
+					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
+					Message(b"hello world".to_vec()),
+				],
+			),
+			(
+				ParaId::from(5),
+				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
+			),
+		];
+
+		let roots: HashMap<_, _> = actual_messages.iter()
+			.map(|&(para_id, ref messages)| (
+				para_id,
+				::polkadot_validation::message_queue_root(messages.iter().map(|m| &m.0)),
+			))
+			.collect();
+
+		let inputs = [
+			(
+				ParaId::from(1), // wrong message.
+				vec![Message(vec![1, 1, 2, 2]), Message(vec![3, 3, 4, 4])],
+			),
+			(
+				ParaId::from(1),
+				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
+			),
+			(
+				ParaId::from(1), // duplicate
+				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
+			),
+
+			(
+				ParaId::from(5), // out of order
+				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
+			),
+			(
+				ParaId::from(1234), // un-routed parachain.
+				vec![Message(vec![9, 9, 9, 9])],
+			),
+			(
+				ParaId::from(2),
+				vec![
+					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
+					Message(b"hello world".to_vec()),
+				],
+			),
+		];
+		let ingress = ComputeIngress {
+			ingress_roots: roots,
+			incoming: Vec::new(),
+			inner: stream::iter_ok::<_, ()>(inputs.iter().cloned()),
+		};
+
+		assert_eq!(ingress.wait().unwrap().unwrap(), actual_messages);
+	}
+
+	#[test]
+	fn add_new_sessions_works() {
+		let mut live_sessions = LiveValidationSessions::new();
+		let key_a: ValidatorId = [0; 32].unchecked_into();
+		let key_b: ValidatorId = [1; 32].unchecked_into();
+		let parent_hash = [0xff; 32].into();
+
+		let (session, new_key) = live_sessions.new_validation_session(SessionParams {
+			parent_hash,
+			local_session_key: None,
+			authorities: Vec::new(),
+		});
+
+		let knowledge = session.knowledge().clone();
+
+		assert!(new_key.is_none());
+
+		let (session, new_key) = live_sessions.new_validation_session(SessionParams {
+			parent_hash,
+			local_session_key: Some(key_a.clone()),
+			authorities: Vec::new(),
+		});
+
+		// check that knowledge points to the same place.
+		assert_eq!(&**session.knowledge() as *const _, &*knowledge as *const _);
+		assert_eq!(new_key, Some(key_a.clone()));
+
+		let (session, new_key) = live_sessions.new_validation_session(SessionParams {
+			parent_hash,
+			local_session_key: Some(key_b.clone()),
+			authorities: Vec::new(),
+		});
+
+		assert_eq!(&**session.knowledge() as *const _, &*knowledge as *const _);
+		assert!(new_key.is_none());
+	}
 }

polkadot/service/src/lib.rs

@@ -212,6 +212,13 @@ construct_service_factory! {
 					None => return Ok(service),
 				};
 
+				if service.config.custom.collating_for.is_some() {
+					info!("The node cannot start as an authority because it is also configured\
+						to run as a collator.");
+
+					return Ok(service);
+				}
+
 				let client = service.client();
 				let known_oracle = client.clone();
 

polkadot/src/main.rs

@@ -25,7 +25,7 @@ extern crate futures;
 #[macro_use]
 extern crate error_chain;
 
-use cli::{PolkadotService, VersionInfo};
+use cli::{PolkadotService, VersionInfo, TaskExecutor};
 use futures::sync::oneshot;
 use futures::{future, Future};
 
@@ -52,7 +52,7 @@ impl cli::IntoExit for Worker {
 
 impl cli::Worker for Worker {
 	type Work = <Self as cli::IntoExit>::Exit;
-	fn work<S: PolkadotService>(self, _service: &S) -> Self::Work {
+	fn work<S: PolkadotService>(self, _service: &S, _: TaskExecutor) -> Self::Work {
 		use cli::IntoExit;
 		self.into_exit()
 	}

polkadot/test-parachains/adder/collator/src/main.rs

@@ -33,7 +33,7 @@ use std::sync::Arc;
 use adder::{HeadData as AdderHead, BlockData as AdderBody};
 use substrate_primitives::{Pair as PairT, ed25519::Pair};
 use parachain::codec::{Encode, Decode};
-use primitives::parachain::{HeadData, BlockData, Id as ParaId, Message};
+use primitives::parachain::{HeadData, BlockData, Id as ParaId, Message, Extrinsic};
 use collator::{InvalidHead, ParachainContext, VersionInfo};
 use parking_lot::Mutex;
 
@@ -59,7 +59,7 @@ impl ParachainContext for AdderContext {
 		&self,
 		last_head: HeadData,
 		ingress: I,
-	) -> Result<(BlockData, HeadData), InvalidHead>
+	) -> Result<(BlockData, HeadData, Extrinsic), InvalidHead>
 	{
 		let adder_head = AdderHead::decode(&mut &last_head.0[..])
 			.ok_or(InvalidHead)?;
@@ -93,7 +93,7 @@ impl ParachainContext for AdderContext {
 			next_head.number, next_body.state.overflowing_add(next_body.add).0);
 
 		db.insert(next_head.clone(), next_body);
-		Ok((encoded_body, encoded_head))
+		Ok((encoded_body, encoded_head, Extrinsic { outgoing_messages: Vec::new() }))
 	}
 }
 

polkadot/validation/src/attestation_service.rs

@@ -118,6 +118,7 @@ pub(crate) fn start<C, N, P>(
 		N: Network + Send + Sync + 'static,
 		N::TableRouter: Send + 'static,
 		<<N::TableRouter as TableRouter>::FetchIncoming as IntoFuture>::Future: Send + 'static,
+		<N::BuildTableRouter as IntoFuture>::Future: Send + 'static,
 {
 	const TIMER_DELAY: Duration = Duration::from_secs(5);
 	const TIMER_INTERVAL: Duration = Duration::from_secs(30);

polkadot/validation/src/collation.rs

@@ -170,7 +170,7 @@ pub fn message_queue_root<A, I: IntoIterator<Item=A>>(messages: I) -> Hash
 }
 
 /// Compute the set of egress roots for all given outgoing messages.
-pub fn egress_roots(mut outgoing: Vec<OutgoingMessage>) -> Vec<(ParaId, Hash)> {
+pub fn egress_roots(outgoing: &mut [OutgoingMessage]) -> Vec<(ParaId, Hash)> {
 	// stable sort messages by parachain ID.
 	outgoing.sort_by_key(|msg| ParaId::from(msg.target));
 
@@ -357,7 +357,7 @@ mod tests {
 			&[(1.into(), root_1), (2.into(), root_2), (3.into(), root_3)],
 		).is_ok());
 
-		let egress_roots = egress_roots(messages.clone());
+		let egress_roots = egress_roots(&mut messages.clone()[..]);
 
 		assert!(check_extrinsic(
 			messages.clone(),

polkadot/validation/src/lib.rs

@@ -167,10 +167,17 @@ pub trait TableRouter: Clone {
 
 /// A long-lived network which can create parachain statement and BFT message routing processes on demand.
 pub trait Network {
+	/// The error type of asynchronously building the table router.
+	type Error: std::fmt::Debug;
+
 	/// The table router type. This should handle importing of any statements,
 	/// routing statements to peers, and driving completion of any `StatementProducers`.
 	type TableRouter: TableRouter;
 
+	/// The future used for asynchronously building the table router.
+	/// This should not fail.
+	type BuildTableRouter: IntoFuture<Item=Self::TableRouter,Error=Self::Error>;
+
 	/// Instantiate a table router using the given shared table.
 	/// Also pass through any outgoing messages to be broadcast to peers.
 	fn communication_for(
@@ -178,7 +185,7 @@ pub trait Network {
 		table: Arc<SharedTable>,
 		outgoing: Outgoing,
 		authorities: &[SessionKey],
-	) -> Self::TableRouter;
+	) -> Self::BuildTableRouter;
 }
 
 /// Information about a specific group.
@@ -284,6 +291,7 @@ impl<C, N, P> ParachainValidation<C, N, P> where
 	<C::Collation as IntoFuture>::Future: Send + 'static,
 	N::TableRouter: Send + 'static,
 	<<N::TableRouter as TableRouter>::FetchIncoming as IntoFuture>::Future: Send + 'static,
+	<N::BuildTableRouter as IntoFuture>::Future: Send + 'static,
 {
 	/// Get an attestation table for given parent hash.
 	///
@@ -382,19 +390,21 @@ impl<C, N, P> ParachainValidation<C, N, P> where
 		&self,
 		relay_parent: Hash,
 		validation_para: ParaId,
-		router: N::TableRouter,
+		build_router: N::BuildTableRouter
 	) -> exit_future::Signal {
 		use extrinsic_store::Data;
 
 		let (signal, exit) = exit_future::signal();
+		let (collators, client) = (self.collators.clone(), self.client.clone());
+		let extrinsic_store = self.extrinsic_store.clone();
 
+		let with_router = move |router: N::TableRouter| {
 			let fetch_incoming = router.fetch_incoming(validation_para)
 				.into_future()
 				.map_err(|e| format!("{:?}", e));
 
 			// fetch incoming messages to our parachain from network and
 			// then fetch a local collation.
-		let (collators, client) = (self.collators.clone(), self.client.clone());
 			let collation_work = fetch_incoming
 				.map_err(|e| String::clone(&e))
 				.and_then(move |incoming| {
@@ -407,8 +417,7 @@ impl<C, N, P> ParachainValidation<C, N, P> where
 					).map_err(|e| format!("{:?}", e))
 				});
 
-		let extrinsic_store = self.extrinsic_store.clone();
+			collation_work.then(move |result| match result {
-		let handled_work = collation_work.then(move |result| match result {
 				Ok((collation, extrinsic)) => {
 					let res = extrinsic_store.make_available(Data {
 						relay_parent,
@@ -422,7 +431,11 @@ impl<C, N, P> ParachainValidation<C, N, P> where
 						Ok(()) => {
 							// TODO: https://github.com/paritytech/polkadot/issues/51
 							// Erasure-code and provide merkle branches.
-						router.local_candidate(collation.receipt, collation.block_data, extrinsic)
+							router.local_candidate(
+								collation.receipt,
+								collation.block_data,
+								extrinsic,
+							)
 						}
 						Err(e) => warn!(
 							target: "validation",
@@ -437,9 +450,17 @@ impl<C, N, P> ParachainValidation<C, N, P> where
 					warn!(target: "validation", "Failed to collate candidate: {}", e);
 					Ok(())
 				}
-		});
+			})
+		};
 
-		let cancellable_work = handled_work.select(exit).then(|_| Ok(()));
+		let cancellable_work = build_router
+			.into_future()
+			.map_err(|e| {
+				warn!(target: "validation" , "Failed to build table router: {:?}", e);
+			})
+			.and_then(with_router)
+			.select(exit)
+			.then(|_| Ok(()));
 
 		// spawn onto thread pool.
 		self.handle.spawn(cancellable_work);
@@ -472,6 +493,7 @@ impl<C, N, P, TxApi> ProposerFactory<C, N, P, TxApi> where
 	N: Network + Send + Sync + 'static,
 	N::TableRouter: Send + 'static,
 	<<N::TableRouter as TableRouter>::FetchIncoming as IntoFuture>::Future: Send + 'static,
+	<N::BuildTableRouter as IntoFuture>::Future: Send + 'static,
 	TxApi: PoolChainApi,
 {
 	/// Create a new proposer factory.
@@ -521,6 +543,7 @@ impl<C, N, P, TxApi> consensus::Environment<Block> for ProposerFactory<C, N, P,
 	<C::Collation as IntoFuture>::Future: Send + 'static,
 	N::TableRouter: Send + 'static,
 	<<N::TableRouter as TableRouter>::FetchIncoming as IntoFuture>::Future: Send + 'static,
+	<N::BuildTableRouter as IntoFuture>::Future: Send + 'static,
 {
 	type Proposer = Proposer<P, TxApi>;
 	type Error = Error;