Minimal parachains part 2: Parachain statement and data routing (#173)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* schedule collation work as soon as BFT is started

* impl future in collator

* fix comment

* governance proposals for adding and removing parachains

* bump protocol version

* tear out polkadot-specific pieces of substrate-network

* extract out polkadot-specific stuff from substrate-network

* begin polkadot network subsystem

* grumbles

* update WASM checkins

* parse status from polkadot peer

* allow invoke of network specialization

* begin statement router implementation

* remove dependency on tokio-timer

* fix sanity check and have proposer factory create communication streams

* pull out statement routing from consensus library

* fix comments

* adjust typedefs

* extract consensus_gossip out of main network protocol handler

* port substrate-bft to new tokio

* port polkadot-consensus to new tokio

* fix typo

* start message processing task

* initial consensus network implementation

* remove known tracking from statement-table crate

* extract router into separate module

* defer statements until later

* double signature is invalid

* propagating statements

* grumbles

* request block data

* fix compilation

* embed new consensus network into service

* port demo CLI to tokio

* all test crates compile

* some tests for fetching block data

* whitespace

* adjusting some tokio stuff

* update exit-future

* remove overly noisy warning

* clean up collation work a bit

* address review grumbles

* fix lock order in protocol handler

* rebuild wasm artifacts

* tag AuthorityId::from_slice for std only

* address formatting grumbles

* rename event_loop to executor

* some more docs for polkadot-network crate

polkadot/service/src/components.rs

@@ -18,20 +18,22 @@
 
 use std::collections::HashMap;
 use std::sync::Arc;
-use client::{self, Client};
+use chain_spec::ChainSpec;
 use client_db;
+use client::{self, Client};
 use codec::{self, Slicable};
 use consensus;
+use error;
 use keystore::Store as Keystore;
-use network;
+use network::{self, OnDemand};
 use polkadot_api;
 use polkadot_executor::Executor as LocalDispatch;
+use polkadot_network::NetworkService;
 use polkadot_primitives::{Block, BlockId, Hash};
 use state_machine;
 use substrate_executor::NativeExecutor;
 use transaction_pool::{self, TransactionPool};
-use error;
-use chain_spec::ChainSpec;
+use tokio::runtime::TaskExecutor;
 
 /// Code executor.
 pub type CodeExecutor = NativeExecutor<LocalDispatch>;
@@ -49,7 +51,7 @@ pub trait Components {
 
 	/// Create client.
 	fn build_client(&self, settings: client_db::DatabaseSettings, executor: CodeExecutor, chain_spec: &ChainSpec)
-		-> Result<(Arc<Client<Self::Backend, Self::Executor, Block>>, Option<Arc<network::OnDemand<Block, network::Service<Block>>>>), error::Error>;
+		-> Result<(Arc<Client<Self::Backend, Self::Executor, Block>>, Option<Arc<OnDemand<Block, NetworkService>>>), error::Error>;
 
 	/// Create api.
 	fn build_api(&self, client: Arc<Client<Self::Backend, Self::Executor, Block>>) -> Arc<Self::Api>;
@@ -59,7 +61,14 @@ pub trait Components {
 		-> Arc<network::TransactionPool<Block>>;
 
 	/// Create consensus service.
-	fn build_consensus(&self, client: Arc<Client<Self::Backend, Self::Executor, Block>>, network: Arc<network::Service<Block>>, tx_pool: Arc<TransactionPool<Self::Api>>, keystore: &Keystore)
+	fn build_consensus(
+		&self,
+		client: Arc<Client<Self::Backend, Self::Executor, Block>>,
+		network: Arc<NetworkService>,
+		tx_pool: Arc<TransactionPool<Self::Api>>,
+		keystore: &Keystore,
+		task_executor: TaskExecutor,
+	)
 		-> Result<Option<consensus::Service>, error::Error>;
 }
 
@@ -75,7 +84,7 @@ impl Components for FullComponents {
 	type Executor = client::LocalCallExecutor<client_db::Backend<Block>, NativeExecutor<LocalDispatch>>;
 
 	fn build_client(&self, db_settings: client_db::DatabaseSettings, executor: CodeExecutor, chain_spec: &ChainSpec)
-		-> Result<(Arc<client::Client<Self::Backend, Self::Executor, Block>>, Option<Arc<network::OnDemand<Block, network::Service<Block>>>>), error::Error> {
+		-> Result<(Arc<client::Client<Self::Backend, Self::Executor, Block>>, Option<Arc<OnDemand<Block, NetworkService>>>), error::Error> {
 		Ok((Arc::new(client_db::new_client(db_settings, executor, chain_spec)?), None))
 	}
 
@@ -92,20 +101,31 @@ impl Components for FullComponents {
 		})
 	}
 
-	fn build_consensus(&self, client: Arc<client::Client<Self::Backend, Self::Executor, Block>>, network: Arc<network::Service<Block>>, tx_pool: Arc<TransactionPool<Self::Api>>, keystore: &Keystore)
-		-> Result<Option<consensus::Service>, error::Error> {
+	fn build_consensus(
+		&self,
+		client: Arc<client::Client<Self::Backend, Self::Executor, Block>>,
+		network: Arc<NetworkService>,
+		tx_pool: Arc<TransactionPool<Self::Api>>,
+		keystore: &Keystore,
+		task_executor: TaskExecutor,
+	)
+		-> Result<Option<consensus::Service>, error::Error>
+	{
 		if !self.is_validator {
 			return Ok(None);
 		}
 
 		// Load the first available key
 		let key = keystore.load(&keystore.contents()?[0], "")?;
-		info!("Using authority key: {}", key.public());
+		info!("Using authority key {}", key.public());
+
+		let consensus_net = ::polkadot_network::consensus::ConsensusNetwork::new(network, client.clone());
 		Ok(Some(consensus::Service::new(
 			client.clone(),
 			client.clone(),
-			network.clone(),
+			consensus_net,
 			tx_pool.clone(),
+			task_executor,
 			::std::time::Duration::from_millis(4000), // TODO: dynamic
 			key,
 		)))
@@ -116,14 +136,14 @@ impl Components for FullComponents {
 pub struct LightComponents;
 
 impl Components for LightComponents {
-	type Backend = client::light::backend::Backend<client_db::light::LightStorage<Block>, network::OnDemand<Block, network::Service<Block>>>;
+	type Backend = client::light::backend::Backend<client_db::light::LightStorage<Block>, network::OnDemand<Block, NetworkService>>;
 	type Api = polkadot_api::light::RemotePolkadotApiWrapper<Self::Backend, Self::Executor>;
 	type Executor = client::light::call_executor::RemoteCallExecutor<
-		client::light::blockchain::Blockchain<client_db::light::LightStorage<Block>, network::OnDemand<Block, network::Service<Block>>>,
-		network::OnDemand<Block, network::Service<Block>>>;
+		client::light::blockchain::Blockchain<client_db::light::LightStorage<Block>, network::OnDemand<Block, NetworkService>>,
+		network::OnDemand<Block, NetworkService>>;
 
 	fn build_client(&self, db_settings: client_db::DatabaseSettings, executor: CodeExecutor, spec: &ChainSpec)
-		-> Result<(Arc<client::Client<Self::Backend, Self::Executor, Block>>, Option<Arc<network::OnDemand<Block, network::Service<Block>>>>), error::Error> {
+		-> Result<(Arc<client::Client<Self::Backend, Self::Executor, Block>>, Option<Arc<OnDemand<Block, NetworkService>>>), error::Error> {
 		let db_storage = client_db::light::LightStorage::new(db_settings)?;
 		let light_blockchain = client::light::new_light_blockchain(db_storage);
 		let fetch_checker = Arc::new(client::light::new_fetch_checker(light_blockchain.clone(), executor));
@@ -146,8 +166,16 @@ impl Components for LightComponents {
 		})
 	}
 
-	fn build_consensus(&self, _client: Arc<client::Client<Self::Backend, Self::Executor, Block>>, _network: Arc<network::Service<Block>>, _tx_pool: Arc<TransactionPool<Self::Api>>, _keystore: &Keystore)
-		-> Result<Option<consensus::Service>, error::Error> {
+	fn build_consensus(
+		&self,
+		_client: Arc<client::Client<Self::Backend, Self::Executor, Block>>,
+		_network: Arc<NetworkService>,
+		_tx_pool: Arc<TransactionPool<Self::Api>>,
+		_keystore: &Keystore,
+		_task_executor: TaskExecutor,
+	)
+		-> Result<Option<consensus::Service>, error::Error>
+	{
 		Ok(None)
 	}
 }

polkadot/service/src/lib.rs

@@ -21,7 +21,6 @@ extern crate futures;
 extern crate ed25519;
 extern crate clap;
 extern crate exit_future;
-extern crate tokio_timer;
 extern crate serde;
 extern crate serde_json;
 extern crate polkadot_primitives;
@@ -30,18 +29,17 @@ extern crate polkadot_executor;
 extern crate polkadot_api;
 extern crate polkadot_consensus as consensus;
 extern crate polkadot_transaction_pool as transaction_pool;
+extern crate polkadot_network;
 extern crate substrate_keystore as keystore;
-extern crate substrate_runtime_io as runtime_io;
 extern crate substrate_primitives as primitives;
 extern crate substrate_runtime_primitives as runtime_primitives;
 extern crate substrate_network as network;
 extern crate substrate_codec as codec;
 extern crate substrate_executor;
 extern crate substrate_state_machine as state_machine;
-
-extern crate tokio_core;
 extern crate substrate_client as client;
 extern crate substrate_client_db as client_db;
+extern crate tokio;
 
 #[macro_use]
 extern crate substrate_telemetry;
@@ -62,9 +60,7 @@ mod config;
 mod chain_spec;
 
 use std::sync::Arc;
-use std::thread;
 use futures::prelude::*;
-use tokio_core::reactor::Core;
 use transaction_pool::TransactionPool;
 use keystore::Store as Keystore;
 use polkadot_api::PolkadotApi;
@@ -72,6 +68,8 @@ use polkadot_primitives::{Block, BlockId, Hash};
 use client::{Client, BlockchainEvents};
 use network::ManageNetwork;
 use exit_future::Signal;
+use polkadot_network::{NetworkService, PolkadotProtocol};
+use tokio::runtime::TaskExecutor;
 
 pub use self::error::{ErrorKind, Error};
 pub use self::components::{Components, FullComponents, LightComponents};
@@ -80,23 +78,22 @@ pub use chain_spec::ChainSpec;
 
 /// Polkadot service.
 pub struct Service<Components: components::Components> {
-	thread: Option<thread::JoinHandle<()>>,
 	client: Arc<Client<Components::Backend, Components::Executor, Block>>,
-	network: Arc<network::Service<Block>>,
+	network: Arc<NetworkService>,
 	transaction_pool: Arc<TransactionPool<Components::Api>>,
 	signal: Option<Signal>,
 	_consensus: Option<consensus::Service>,
 }
 
 /// Creates light client and register protocol with the network service
-pub fn new_light(config: Configuration) -> Result<Service<components::LightComponents>, error::Error> {
-	Service::new(components::LightComponents, config)
+pub fn new_light(config: Configuration, executor: TaskExecutor) -> Result<Service<components::LightComponents>, error::Error> {
+	Service::new(components::LightComponents, config, executor)
 }
 
 /// Creates full client and register protocol with the network service
-pub fn new_full(config: Configuration) -> Result<Service<components::FullComponents>, error::Error> {
-	let is_validator = (config.roles & Role::VALIDATOR) == Role::VALIDATOR;
-	Service::new(components::FullComponents { is_validator }, config)
+pub fn new_full(config: Configuration, executor: TaskExecutor) -> Result<Service<components::FullComponents>, error::Error> {
+	let is_validator = (config.roles & Role::AUTHORITY) == Role::AUTHORITY;
+	Service::new(components::FullComponents { is_validator }, config, executor)
 }
 
 /// Creates bare client without any networking.
@@ -112,7 +109,7 @@ pub fn new_client(config: Configuration) -> Result<Arc<Client<
 		pruning: config.pruning,
 	};
 	let executor = polkadot_executor::Executor::new();
-	let is_validator = (config.roles & Role::VALIDATOR) == Role::VALIDATOR;
+	let is_validator = (config.roles & Role::AUTHORITY) == Role::AUTHORITY;
 	let components = components::FullComponents { is_validator };
 	let (client, _) = components.build_client(db_settings, executor, &config.chain_spec)?;
 	Ok(client)
@@ -124,9 +121,7 @@ impl<Components> Service<Components>
 		client::error::Error: From<<<<Components as components::Components>::Backend as client::backend::Backend<Block>>::State as state_machine::Backend>::Error>,
 {
 	/// Creates and register protocol with the network service
-	fn new(components: Components, config: Configuration) -> Result<Self, error::Error> {
-		use std::sync::Barrier;
-
+	fn new(components: Components, config: Configuration, task_executor: TaskExecutor) -> Result<Self, error::Error> {
 		let (signal, exit) = ::exit_future::signal();
 
 		// Create client
@@ -165,61 +160,55 @@ impl<Components> Service<Components>
 			chain: client.clone(),
 			on_demand: on_demand.clone().map(|d| d as Arc<network::OnDemandService<Block>>),
 			transaction_pool: transaction_pool_adapter,
+			specialization: PolkadotProtocol::new(),
 		};
-		let network = network::Service::new(network_params)?;
-		let barrier = ::std::sync::Arc::new(Barrier::new(2));
+
+		let network = network::Service::new(network_params, ::polkadot_network::DOT_PROTOCOL_ID)?;
 		on_demand.map(|on_demand| on_demand.set_service_link(Arc::downgrade(&network)));
 
-		let thread = {
-			let client = client.clone();
+		network.start_network();
+
+		{
+			// block notifications
 			let network = network.clone();
 			let txpool = transaction_pool.clone();
 
-			let thread_barrier = barrier.clone();
-			thread::spawn(move || {
-				network.start_network();
+			let events = client.import_notification_stream()
+				.for_each(move |notification| {
+					network.on_block_imported(notification.hash, &notification.header);
+					prune_imported(&*txpool, notification.hash);
+					Ok(())
+				})
+				.select(exit.clone())
+				.then(|_| Ok(()));
+			task_executor.spawn(events);
+		}
 
-				thread_barrier.wait();
-				let mut core = Core::new().expect("tokio::Core could not be created");
+		{
+			// transaction notifications
+			let network = network.clone();
+			let events = transaction_pool.import_notification_stream()
+				// TODO [ToDr] Consider throttling?
+				.for_each(move |_| {
+					network.trigger_repropagate();
+					Ok(())
+				})
+				.select(exit.clone())
+				.then(|_| Ok(()));
 
-				// block notifications
-				let network1 = network.clone();
-				let txpool1 = txpool.clone();
-
-				let events = client.import_notification_stream()
-					.for_each(move |notification| {
-						network1.on_block_imported(notification.hash, &notification.header);
-						prune_imported(&*txpool1, notification.hash);
-
-						Ok(())
-					});
-				core.handle().spawn(events);
-
-				// transaction notifications
-				let events = txpool.import_notification_stream()
-					// TODO [ToDr] Consider throttling?
-					.for_each(move |_| {
-						network.trigger_repropagate();
-						Ok(())
-					});
-				core.handle().spawn(events);
-
-				if let Err(e) = core.run(exit) {
-					debug!("Polkadot service event loop shutdown with {:?}", e);
-				}
-				debug!("Polkadot service shutdown");
-			})
-		};
-
-		// wait for the network to start up before starting the consensus
-		// service.
-		barrier.wait();
+			task_executor.spawn(events);
+		}
 
 		// Spin consensus service if configured
-		let consensus_service = components.build_consensus(client.clone(), network.clone(), transaction_pool.clone(), &keystore)?;
+		let consensus_service = components.build_consensus(
+			client.clone(),
+			network.clone(),
+			transaction_pool.clone(),
+			&keystore,
+			task_executor,
+		)?;
 
 		Ok(Service {
-			thread: Some(thread),
 			client: client,
 			network: network,
 			transaction_pool: transaction_pool,
@@ -234,7 +223,7 @@ impl<Components> Service<Components>
 	}
 
 	/// Get shared network instance.
-	pub fn network(&self) -> Arc<network::Service<Block>> {
+	pub fn network(&self) -> Arc<NetworkService> {
 		self.network.clone()
 	}
 
@@ -260,14 +249,12 @@ pub fn prune_imported<A>(pool: &TransactionPool<A>, hash: Hash)
 
 impl<Components> Drop for Service<Components> where Components: components::Components {
 	fn drop(&mut self) {
+		debug!(target: "service", "Polkadot service shutdown");
+
 		self.network.stop_network();
 
 		if let Some(signal) = self.signal.take() {
 			signal.fire();
 		}
-
-		if let Some(thread) = self.thread.take() {
-			thread.join().expect("The service thread has panicked");
-		}
 	}
 }