Collators get incoming parachain messages (#149)

* refactor out a consensus data fetcher from table router

* move statement checking logic into router

* refuse to start authority if collator

* support building the table router asynchronously

* instantiate_consensus does not overwrite old

* update key in new consensus if there was none before

* collator collects ingress from network

* test produced egress roots

* fix adder-collator compilation

* address first grumbles

* integrate new gossip with collator network launch

* address review

polkadot/network/src/router.rs

@@ -14,10 +14,11 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Statement routing and consensus table router implementation.
+//! Statement routing and validation statement table router implementation.
 //!
-//! During the consensus process, validators exchange statements on validity and availability
+//! During the attestation process, validators exchange statements on validity and availability
 //! of parachain candidates.
+//!
 //! The `Router` in this file hooks into the underlying network to fulfill
 //! the `TableRouter` trait from `polkadot-validation`, which is expected to call into a shared statement table
 //! and dispatch evaluation work as necessary when new statements come in.
@@ -33,18 +34,15 @@ use polkadot_primitives::parachain::{
 };
 
 use codec::{Encode, Decode};
-use futures::{future, prelude::*};
-use futures::sync::oneshot::{self, Receiver};
+use futures::prelude::*;
 use parking_lot::Mutex;
 
-use std::collections::{hash_map::{Entry, HashMap}, HashSet};
-use std::{io, mem};
+use std::collections::{HashMap, HashSet};
+use std::io;
 use std::sync::Arc;
 
-use gossip::{RegisteredMessageValidator};
-use validation::{NetworkService, Knowledge, Executor};
+use validation::{self, SessionDataFetcher, NetworkService, Executor};
 
-type IngressPair = (ParaId, Vec<Message>);
 type IngressPairRef<'a> = (ParaId, &'a [Message]);
 
 /// Compute the gossip topic for attestations on the given parent hash.
@@ -55,109 +53,48 @@ pub(crate) fn attestation_topic(parent_hash: Hash) -> Hash {
 	BlakeTwo256::hash(&v[..])
 }
 
-fn incoming_message_topic(parent_hash: Hash, parachain: ParaId) -> Hash {
-	let mut v = parent_hash.as_ref().to_vec();
-	parachain.using_encoded(|s| v.extend(s));
-	v.extend(b"incoming");
-
-	BlakeTwo256::hash(&v[..])
-}
-
-/// Receiver for block data.
-pub struct BlockDataReceiver {
-	outer: Receiver<Receiver<BlockData>>,
-	inner: Option<Receiver<BlockData>>
-}
-
-impl Future for BlockDataReceiver {
-	type Item = BlockData;
-	type Error = io::Error;
-
-	fn poll(&mut self) -> Poll<BlockData, io::Error> {
-		let map_err = |_| io::Error::new(
-			io::ErrorKind::Other,
-			"Sending end of channel hung up",
-		);
-
-		if let Some(ref mut inner) = self.inner {
-			return inner.poll().map_err(map_err);
-		}
-		match self.outer.poll().map_err(map_err)? {
-			Async::Ready(mut inner) => {
-				let poll_result = inner.poll();
-				self.inner = Some(inner);
-				poll_result.map_err(map_err)
-			}
-			Async::NotReady => Ok(Async::NotReady),
-		}
-	}
-}
-/// receiver for incoming data.
-#[derive(Clone)]
-pub struct IncomingReceiver {
-	inner: future::Shared<Receiver<Incoming>>
-}
-
-impl Future for IncomingReceiver {
-	type Item = Incoming;
-	type Error = io::Error;
-
-	fn poll(&mut self) -> Poll<Incoming, io::Error> {
-		match self.inner.poll() {
-			Ok(Async::NotReady) => Ok(Async::NotReady),
-			Ok(Async::Ready(i)) => Ok(Async::Ready(Incoming::clone(&*i))),
-			Err(_) => Err(io::Error::new(
-				io::ErrorKind::Other,
-				"Sending end of channel hung up",
-			)),
-		}
-	}
-}
-
 /// Table routing implementation.
 pub struct Router<P, E, N: NetworkService, T> {
 	table: Arc<SharedTable>,
-	network: Arc<N>,
-	api: Arc<P>,
-	exit: E,
-	task_executor: T,
-	parent_hash: Hash,
 	attestation_topic: Hash,
-	knowledge: Arc<Mutex<Knowledge>>,
-	fetch_incoming: Arc<Mutex<HashMap<ParaId, IncomingReceiver>>>,
+	fetcher: SessionDataFetcher<P, E, N, T>,
 	deferred_statements: Arc<Mutex<DeferredStatements>>,
-	message_validator: RegisteredMessageValidator,
 }
 
 impl<P, E, N: NetworkService, T> Router<P, E, N, T> {
 	pub(crate) fn new(
 		table: Arc<SharedTable>,
-		network: Arc<N>,
-		api: Arc<P>,
-		task_executor: T,
-		parent_hash: Hash,
-		knowledge: Arc<Mutex<Knowledge>>,
-		exit: E,
-		message_validator: RegisteredMessageValidator,
+		fetcher: SessionDataFetcher<P, E, N, T>,
 	) -> Self {
+		let parent_hash = fetcher.parent_hash();
 		Router {
 			table,
-			network,
-			api,
-			task_executor,
-			parent_hash,
 			attestation_topic: attestation_topic(parent_hash),
-			knowledge,
-			fetch_incoming: Arc::new(Mutex::new(HashMap::new())),
 			deferred_statements: Arc::new(Mutex::new(DeferredStatements::new())),
-			exit,
-			message_validator,
+			fetcher,
 		}
 	}
 
-	/// Get the attestation topic for gossip.
-	pub(crate) fn gossip_topic(&self) -> Hash {
-		self.attestation_topic
+	/// Return a future of checked messages. These should be imported into the router
+	/// with `import_statement`.
+	pub(crate) fn checked_statements(&self) -> impl Stream<Item=SignedStatement,Error=()> {
+		// spin up a task in the background that processes all incoming statements
+		// validation has been done already by the gossip validator.
+		// this will block internally until the gossip messages stream is obtained.
+		self.network().gossip_messages_for(self.attestation_topic)
+			.filter_map(|msg| {
+				debug!(target: "validation", "Processing statement for live validation session");
+				crate::gossip::GossipMessage::decode(&mut &msg[..])
+			})
+			.map(|msg| msg.statement)
+	}
+
+	fn parent_hash(&self) -> Hash {
+		self.fetcher.parent_hash()
+	}
+
+	fn network(&self) -> &Arc<N> {
+		self.fetcher.network()
 	}
 }
 
@@ -165,16 +102,9 @@ impl<P, E: Clone, N: NetworkService, T: Clone> Clone for Router<P, E, N, T> {
 	fn clone(&self) -> Self {
 		Router {
 			table: self.table.clone(),
-			network: self.network.clone(),
-			api: self.api.clone(),
-			task_executor: self.task_executor.clone(),
-			parent_hash: self.parent_hash.clone(),
+			fetcher: self.fetcher.clone(),
 			attestation_topic: self.attestation_topic.clone(),
 			deferred_statements: self.deferred_statements.clone(),
-			fetch_incoming: self.fetch_incoming.clone(),
-			knowledge: self.knowledge.clone(),
-			exit: self.exit.clone(),
-			message_validator: self.message_validator.clone(),
 		}
 	}
 }
@@ -187,7 +117,7 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 {
 	/// Import a statement whose signature has been checked already.
 	pub(crate) fn import_statement(&self, statement: SignedStatement) {
-		trace!(target: "p_net", "importing consensus statement {:?}", statement.statement);
+		trace!(target: "p_net", "importing validation statement {:?}", statement.statement);
 
 		// defer any statements for which we haven't imported the candidate yet
 		let c_hash = {
@@ -214,7 +144,7 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		};
 
 		// prepend the candidate statement.
-		debug!(target: "consensus", "Importing statements about candidate {:?}", c_hash);
+		debug!(target: "validation", "Importing statements about candidate {:?}", c_hash);
 		statements.insert(0, statement);
 		let producers: Vec<_> = self.table.import_remote_statements(
 			self,
@@ -222,12 +152,12 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		);
 		// dispatch future work as necessary.
 		for (producer, statement) in producers.into_iter().zip(statements) {
-			self.knowledge.lock().note_statement(statement.sender, &statement.statement);
+			self.fetcher.knowledge().lock().note_statement(statement.sender, &statement.statement);
 
 			if let Some(work) = producer.map(|p| self.create_work(c_hash, p)) {
-				trace!(target: "consensus", "driving statement work to completion");
-				let work = work.select2(self.exit.clone()).then(|_| Ok(()));
-				self.task_executor.spawn(work);
+				trace!(target: "validation", "driving statement work to completion");
+				let work = work.select2(self.fetcher.exit().clone()).then(|_| Ok(()));
+				self.fetcher.executor().spawn(work);
 			}
 		}
 	}
@@ -251,14 +181,15 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 				group_messages.clear(); // reuse allocation from previous iterations.
 				group_messages.extend(group.iter().map(|msg| msg.data.clone()).map(Message));
 
-				debug!(target: "consensus", "Circulating messages from {:?} to {:?} at {}",
-					source, target, self.parent_hash);
+				debug!(target: "valdidation", "Circulating messages from {:?} to {:?} at {}",
+					source, target, self.parent_hash());
 
 				// this is the ingress from source to target, with given messages.
-				let target_incoming = incoming_message_topic(self.parent_hash, target);
+				let target_incoming =
+					validation::incoming_message_topic(self.parent_hash(), target);
 				let ingress_for: IngressPairRef = (source, &group_messages[..]);
 
-				self.network.gossip_message(target_incoming, ingress_for.encode());
+				self.network().gossip_message(target_incoming, ingress_for.encode());
 			}
 		}
 	}
@@ -269,11 +200,11 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 		D: Future<Item=(BlockData, Incoming),Error=io::Error> + Send + 'static,
 	{
 		let table = self.table.clone();
-		let network = self.network.clone();
-		let knowledge = self.knowledge.clone();
+		let network = self.network().clone();
+		let knowledge = self.fetcher.knowledge().clone();
 		let attestation_topic = self.attestation_topic.clone();
 
-		producer.prime(self.api.clone())
+		producer.prime(self.fetcher.api().clone())
 			.map(move |validated| {
 				// store the data before broadcasting statements, so other peers can fetch.
 				knowledge.lock().note_candidate(
@@ -289,61 +220,6 @@ impl<P: ProvideRuntimeApi + Send + Sync + 'static, E, N, T> Router<P, E, N, T> w
 			})
 			.map_err(|e| debug!(target: "p_net", "Failed to produce statements: {:?}", e))
 	}
-
-}
-
-impl<P: ProvideRuntimeApi, E, N, T> Router<P, E, N, T> where
-	P::Api: ParachainHost<Block>,
-	N: NetworkService,
-	T: Executor,
-	E: Future<Item=(),Error=()> + Clone + Send + 'static,
-{
-	fn do_fetch_incoming(&self, parachain: ParaId) -> IncomingReceiver {
-		use polkadot_primitives::BlockId;
-		let (tx, rx) = {
-			let mut fetching = self.fetch_incoming.lock();
-			match fetching.entry(parachain) {
-				Entry::Occupied(entry) => return entry.get().clone(),
-				Entry::Vacant(entry) => {
-					// has not been requested yet.
-					let (tx, rx) = oneshot::channel();
-					let rx = IncomingReceiver { inner: rx.shared() };
-					entry.insert(rx.clone());
-
-					(tx, rx)
-				}
-			}
-		};
-
-		let parent_hash = self.parent_hash;
-		let topic = incoming_message_topic(parent_hash, parachain);
-		let gossip_messages = self.network.gossip_messages_for(topic)
-			.map_err(|()| panic!("unbounded receivers do not throw errors; qed"))
-			.filter_map(|msg| IngressPair::decode(&mut msg.as_slice()));
-
-		let canon_roots = self.api.runtime_api().ingress(&BlockId::hash(parent_hash), parachain)
-			.map_err(|e| format!("Cannot fetch ingress for parachain {:?} at {:?}: {:?}",
-				parachain, parent_hash, e)
-			);
-
-		let work = canon_roots.into_future()
-			.and_then(move |ingress_roots| match ingress_roots {
-				None => Err(format!("No parachain {:?} registered at {}", parachain, parent_hash)),
-				Some(roots) => Ok(roots.into_iter().collect())
-			})
-			.and_then(move |ingress_roots| ComputeIngress {
-				inner: gossip_messages,
-				ingress_roots,
-				incoming: Vec::new(),
-			})
-			.map(move |incoming| if let Some(i) = incoming { let _ = tx.send(i); })
-			.select2(self.exit.clone())
-			.then(|_| Ok(()));
-
-		self.task_executor.spawn(work);
-
-		rx
-	}
 }
 
 impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> where
@@ -353,8 +229,8 @@ impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> wh
 	E: Future<Item=(),Error=()> + Clone + Send + 'static,
 {
 	type Error = io::Error;
-	type FetchCandidate = BlockDataReceiver;
-	type FetchIncoming = IncomingReceiver;
+	type FetchCandidate = validation::BlockDataReceiver;
+	type FetchIncoming = validation::IncomingReceiver;
 
 	fn local_candidate(&self, receipt: CandidateReceipt, block_data: BlockData, extrinsic: Extrinsic) {
 		// produce a signed statement
@@ -363,42 +239,22 @@ impl<P: ProvideRuntimeApi + Send, E, N, T> TableRouter for Router<P, E, N, T> wh
 		let statement = self.table.import_validated(validated);
 
 		// give to network to make available.
-		self.knowledge.lock().note_candidate(hash, Some(block_data), Some(extrinsic));
-		self.network.gossip_message(self.attestation_topic, statement.encode());
+		self.fetcher.knowledge().lock().note_candidate(hash, Some(block_data), Some(extrinsic));
+		self.network().gossip_message(self.attestation_topic, statement.encode());
 	}
 
-	fn fetch_block_data(&self, candidate: &CandidateReceipt) -> BlockDataReceiver {
-		let parent_hash = self.parent_hash.clone();
-		let candidate = candidate.clone();
-		let (tx, rx) = ::futures::sync::oneshot::channel();
-		self.network.with_spec(move |spec, ctx| {
-			let inner_rx = spec.fetch_block_data(ctx, &candidate, parent_hash);
-			let _ = tx.send(inner_rx);
-		});
-		BlockDataReceiver { outer: rx, inner: None }
+	fn fetch_block_data(&self, candidate: &CandidateReceipt) -> Self::FetchCandidate {
+		self.fetcher.fetch_block_data(candidate)
 	}
 
 	fn fetch_incoming(&self, parachain: ParaId) -> Self::FetchIncoming {
-		self.do_fetch_incoming(parachain)
+		self.fetcher.fetch_incoming(parachain)
 	}
 }
 
 impl<P, E, N: NetworkService, T> Drop for Router<P, E, N, T> {
 	fn drop(&mut self) {
-		let parent_hash = self.parent_hash.clone();
-		self.network.with_spec(move |spec, _| spec.remove_validation_session(&parent_hash));
-		self.network.drop_gossip(self.attestation_topic);
-
-		{
-			let mut incoming_fetched = self.fetch_incoming.lock();
-			for (para_id, _) in incoming_fetched.drain() {
-				self.network.drop_gossip(incoming_message_topic(
-					self.parent_hash,
-					para_id,
-				));
-			}
-		}
-		self.message_validator.remove_session(&parent_hash);
+		self.fetcher.network().drop_gossip(self.attestation_topic);
 	}
 }
 
@@ -457,63 +313,10 @@ impl DeferredStatements {
 	}
 }
 
-// computes ingress from incoming stream of messages.
-// returns `None` if the stream concludes too early.
-#[must_use = "futures do nothing unless polled"]
-struct ComputeIngress<S> {
-	ingress_roots: HashMap<ParaId, Hash>,
-	incoming: Vec<IngressPair>,
-	inner: S,
-}
-
-impl<S> Future for ComputeIngress<S> where S: Stream<Item=IngressPair> {
-	type Item = Option<Incoming>;
-	type Error = S::Error;
-
-	fn poll(&mut self) -> Poll<Option<Incoming>, Self::Error> {
-		loop {
-			if self.ingress_roots.is_empty() {
-				return Ok(Async::Ready(
-					Some(mem::replace(&mut self.incoming, Vec::new()))
-				))
-			}
-
-			let (para_id, messages) = match try_ready!(self.inner.poll()) {
-				None => return Ok(Async::Ready(None)),
-				Some(next) => next,
-			};
-
-			match self.ingress_roots.entry(para_id) {
-				Entry::Vacant(_) => continue,
-				Entry::Occupied(occupied) => {
-					let canon_root = occupied.get().clone();
-					let messages = messages.iter().map(|m| &m.0[..]);
-					if ::polkadot_validation::message_queue_root(messages) != canon_root {
-						continue;
-					}
-
-					occupied.remove();
-				}
-			}
-
-			let pos = self.incoming.binary_search_by_key(
-				&para_id,
-				|&(id, _)| id,
-			)
-				.err()
-				.expect("incoming starts empty and only inserted when \
-					para_id not inserted before; qed");
-
-			self.incoming.insert(pos, (para_id, messages));
-		}
-	}
-}
-
 #[cfg(test)]
 mod tests {
 	use super::*;
 	use substrate_primitives::crypto::UncheckedInto;
-	use futures::stream;
 	use polkadot_primitives::parachain::ValidatorId;
 
 	#[test]
@@ -556,70 +359,4 @@ mod tests {
 			assert!(traces.is_empty());
 		}
 	}
-
-	#[test]
-	fn compute_ingress_works() {
-		let actual_messages = [
-			(
-				ParaId::from(1),
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-			(
-				ParaId::from(2),
-				vec![
-					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
-					Message(b"hello world".to_vec()),
-				],
-			),
-			(
-				ParaId::from(5),
-				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
-			),
-		];
-
-		let roots: HashMap<_, _> = actual_messages.iter()
-			.map(|&(para_id, ref messages)| (
-				para_id,
-				::polkadot_validation::message_queue_root(messages.iter().map(|m| &m.0)),
-			))
-			.collect();
-
-		let inputs = [
-			(
-				ParaId::from(1), // wrong message.
-				vec![Message(vec![1, 1, 2, 2]), Message(vec![3, 3, 4, 4])],
-			),
-			(
-				ParaId::from(1),
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-			(
-				ParaId::from(1), // duplicate
-				vec![Message(vec![1, 3, 5, 6]), Message(vec![4, 4, 4, 4])],
-			),
-
-			(
-				ParaId::from(5), // out of order
-				vec![Message(vec![1, 2, 3, 4, 5]), Message(vec![6, 9, 6, 9])],
-			),
-			(
-				ParaId::from(1234), // un-routed parachain.
-				vec![Message(vec![9, 9, 9, 9])],
-			),
-			(
-				ParaId::from(2),
-				vec![
-					Message(vec![1, 3, 7, 9, 1, 2, 3, 4, 5, 6]),
-					Message(b"hello world".to_vec()),
-				],
-			),
-		];
-		let ingress = ComputeIngress {
-			ingress_roots: roots,
-			incoming: Vec::new(),
-			inner: stream::iter_ok::<_, ()>(inputs.iter().cloned()),
-		};
-
-		assert_eq!(ingress.wait().unwrap().unwrap(), actual_messages);
-	}
 }