Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/network/src/behaviour.rs

@@ -0,0 +1,192 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::{
+	debug_info, discovery::DiscoveryBehaviour, discovery::DiscoveryOut, DiscoveryNetBehaviour,
+	protocol::event::DhtEvent
+};
+use crate::{ExHashT, specialization::NetworkSpecialization};
+use crate::protocol::{CustomMessageOutcome, Protocol};
+use futures::prelude::*;
+use libp2p::NetworkBehaviour;
+use libp2p::core::{Multiaddr, PeerId, PublicKey};
+use libp2p::kad::record;
+use libp2p::swarm::{NetworkBehaviourAction, NetworkBehaviourEventProcess};
+use libp2p::core::{nodes::Substream, muxing::StreamMuxerBox};
+use log::{debug, warn};
+use sr_primitives::traits::Block as BlockT;
+use std::iter;
+use void;
+
+/// General behaviour of the network. Combines all protocols together.
+#[derive(NetworkBehaviour)]
+#[behaviour(out_event = "BehaviourOut<B>", poll_method = "poll")]
+pub struct Behaviour<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> {
+	/// All the substrate-specific protocols.
+	substrate: Protocol<B, S, H>,
+	/// Periodically pings and identifies the nodes we are connected to, and store information in a
+	/// cache.
+	debug_info: debug_info::DebugInfoBehaviour<Substream<StreamMuxerBox>>,
+	/// Discovers nodes of the network.
+	discovery: DiscoveryBehaviour<Substream<StreamMuxerBox>>,
+
+	/// Queue of events to produce for the outside.
+	#[behaviour(ignore)]
+	events: Vec<BehaviourOut<B>>,
+}
+
+/// Event generated by `Behaviour`.
+pub enum BehaviourOut<B: BlockT> {
+	SubstrateAction(CustomMessageOutcome<B>),
+	Dht(DhtEvent),
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Behaviour<B, S, H> {
+	/// Builds a new `Behaviour`.
+	pub fn new(
+		substrate: Protocol<B, S, H>,
+		user_agent: String,
+		local_public_key: PublicKey,
+		known_addresses: Vec<(PeerId, Multiaddr)>,
+		enable_mdns: bool,
+		allow_private_ipv4: bool,
+	) -> Self {
+		Behaviour {
+			substrate,
+			debug_info: debug_info::DebugInfoBehaviour::new(user_agent, local_public_key.clone()),
+			discovery: DiscoveryBehaviour::new(
+				local_public_key,
+				known_addresses,
+				enable_mdns,
+				allow_private_ipv4
+			),
+			events: Vec::new(),
+		}
+	}
+
+	/// Returns the list of nodes that we know exist in the network.
+	pub fn known_peers(&mut self) -> impl Iterator<Item = &PeerId> {
+		self.discovery.known_peers()
+	}
+
+	/// Adds a hard-coded address for the given peer, that never expires.
+	pub fn add_known_address(&mut self, peer_id: PeerId, addr: Multiaddr) {
+		self.discovery.add_known_address(peer_id, addr)
+	}
+
+	/// Borrows `self` and returns a struct giving access to the information about a node.
+	///
+	/// Returns `None` if we don't know anything about this node. Always returns `Some` for nodes
+	/// we're connected to, meaning that if `None` is returned then we're not connected to that
+	/// node.
+	pub fn node(&self, peer_id: &PeerId) -> Option<debug_info::Node> {
+		self.debug_info.node(peer_id)
+	}
+
+	/// Returns a shared reference to the user protocol.
+	pub fn user_protocol(&self) -> &Protocol<B, S, H> {
+		&self.substrate
+	}
+
+	/// Returns a mutable reference to the user protocol.
+	pub fn user_protocol_mut(&mut self) -> &mut Protocol<B, S, H> {
+		&mut self.substrate
+	}
+
+	/// Start querying a record from the DHT. Will later produce either a `ValueFound` or a `ValueNotFound` event.
+	pub fn get_value(&mut self, key: &record::Key) {
+		self.discovery.get_value(key);
+	}
+
+	/// Starts putting a record into DHT. Will later produce either a `ValuePut` or a `ValuePutFailed` event.
+	pub fn put_value(&mut self, key: record::Key, value: Vec<u8>) {
+		self.discovery.put_value(key, value);
+	}
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> NetworkBehaviourEventProcess<void::Void> for
+Behaviour<B, S, H> {
+	fn inject_event(&mut self, event: void::Void) {
+		void::unreachable(event)
+	}
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> NetworkBehaviourEventProcess<CustomMessageOutcome<B>> for
+Behaviour<B, S, H> {
+	fn inject_event(&mut self, event: CustomMessageOutcome<B>) {
+		self.events.push(BehaviourOut::SubstrateAction(event));
+	}
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> NetworkBehaviourEventProcess<debug_info::DebugInfoEvent>
+	for Behaviour<B, S, H> {
+	fn inject_event(&mut self, event: debug_info::DebugInfoEvent) {
+		let debug_info::DebugInfoEvent::Identified { peer_id, mut info } = event;
+		if !info.protocol_version.contains("substrate") {
+			warn!(target: "sub-libp2p", "Connected to a non-Substrate node: {:?}", info);
+		}
+		if info.listen_addrs.len() > 30 {
+			debug!(target: "sub-libp2p", "Node {:?} has reported more than 30 addresses; \
+				it is identified by {:?} and {:?}", peer_id, info.protocol_version,
+				info.agent_version
+			);
+			info.listen_addrs.truncate(30);
+		}
+		for addr in &info.listen_addrs {
+			self.discovery.add_self_reported_address(&peer_id, addr.clone());
+		}
+		self.substrate.add_discovered_nodes(iter::once(peer_id.clone()));
+	}
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> NetworkBehaviourEventProcess<DiscoveryOut>
+	for Behaviour<B, S, H> {
+	fn inject_event(&mut self, out: DiscoveryOut) {
+		match out {
+			DiscoveryOut::UnroutablePeer(_peer_id) => {
+				// Obtaining and reporting listen addresses for unroutable peers back
+				// to Kademlia is handled by the `Identify` protocol, part of the
+				// `DebugInfoBehaviour`. See the `NetworkBehaviourEventProcess`
+				// implementation for `DebugInfoEvent`.
+			}
+			DiscoveryOut::Discovered(peer_id) => {
+				self.substrate.add_discovered_nodes(iter::once(peer_id));
+			}
+			DiscoveryOut::ValueFound(results) => {
+				self.events.push(BehaviourOut::Dht(DhtEvent::ValueFound(results)));
+			}
+			DiscoveryOut::ValueNotFound(key) => {
+				self.events.push(BehaviourOut::Dht(DhtEvent::ValueNotFound(key)));
+			}
+			DiscoveryOut::ValuePut(key) => {
+				self.events.push(BehaviourOut::Dht(DhtEvent::ValuePut(key)));
+			}
+			DiscoveryOut::ValuePutFailed(key) => {
+				self.events.push(BehaviourOut::Dht(DhtEvent::ValuePutFailed(key)));
+			}
+		}
+	}
+}
+
+impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Behaviour<B, S, H> {
+	fn poll<TEv>(&mut self) -> Async<NetworkBehaviourAction<TEv, BehaviourOut<B>>> {
+		if !self.events.is_empty() {
+			return Async::Ready(NetworkBehaviourAction::GenerateEvent(self.events.remove(0)))
+		}
+
+		Async::NotReady
+	}
+}

substrate/client/network/src/chain.rs

@@ -0,0 +1,169 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Blockchain access trait
+
+use client::Client as SubstrateClient;
+use client_api::error::Error;
+use client_api::{ChangesProof, StorageProof, ClientInfo, CallExecutor};
+use consensus::{BlockImport, BlockStatus, Error as ConsensusError};
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT};
+use sr_primitives::generic::{BlockId};
+use sr_primitives::Justification;
+use primitives::{H256, Blake2Hasher, storage::StorageKey};
+
+/// Local client abstraction for the network.
+pub trait Client<Block: BlockT>: Send + Sync {
+	/// Get blockchain info.
+	fn info(&self) -> ClientInfo<Block>;
+
+	/// Get block status.
+	fn block_status(&self, id: &BlockId<Block>) -> Result<BlockStatus, Error>;
+
+	/// Get block hash by number.
+	fn block_hash(&self, block_number: <Block::Header as HeaderT>::Number) -> Result<Option<Block::Hash>, Error>;
+
+	/// Get block header.
+	fn header(&self, id: &BlockId<Block>) -> Result<Option<Block::Header>, Error>;
+
+	/// Get block body.
+	fn body(&self, id: &BlockId<Block>) -> Result<Option<Vec<Block::Extrinsic>>, Error>;
+
+	/// Get block justification.
+	fn justification(&self, id: &BlockId<Block>) -> Result<Option<Justification>, Error>;
+
+	/// Get block header proof.
+	fn header_proof(&self, block_number: <Block::Header as HeaderT>::Number)
+		-> Result<(Block::Header, StorageProof), Error>;
+
+	/// Get storage read execution proof.
+	fn read_proof(&self, block: &Block::Hash, keys: &[Vec<u8>]) -> Result<StorageProof, Error>;
+
+	/// Get child storage read execution proof.
+	fn read_child_proof(
+		&self,
+		block: &Block::Hash,
+		storage_key: &[u8],
+		keys: &[Vec<u8>],
+	) -> Result<StorageProof, Error>;
+
+	/// Get method execution proof.
+	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, StorageProof), Error>;
+
+	/// Get key changes proof.
+	fn key_changes_proof(
+		&self,
+		first: Block::Hash,
+		last: Block::Hash,
+		min: Block::Hash,
+		max: Block::Hash,
+		storage_key: Option<&StorageKey>,
+		key: &StorageKey
+	) -> Result<ChangesProof<Block::Header>, Error>;
+
+	/// Returns `true` if the given `block` is a descendent of `base`.
+	fn is_descendent_of(&self, base: &Block::Hash, block: &Block::Hash) -> Result<bool, Error>;
+}
+
+/// Finality proof provider.
+pub trait FinalityProofProvider<Block: BlockT>: Send + Sync {
+	/// Prove finality of the block.
+	fn prove_finality(&self, for_block: Block::Hash, request: &[u8]) -> Result<Option<Vec<u8>>, Error>;
+}
+
+impl<Block: BlockT> FinalityProofProvider<Block> for () {
+	fn prove_finality(&self, _for_block: Block::Hash, _request: &[u8]) -> Result<Option<Vec<u8>>, Error> {
+		Ok(None)
+	}
+}
+
+impl<B, E, Block, RA> Client<Block> for SubstrateClient<B, E, Block, RA> where
+	B: client_api::backend::Backend<Block, Blake2Hasher> + Send + Sync + 'static,
+	E: CallExecutor<Block, Blake2Hasher> + Send + Sync + 'static,
+	Self: BlockImport<Block, Error=ConsensusError>,
+	Block: BlockT<Hash=H256>,
+	RA: Send + Sync
+{
+	fn info(&self) -> ClientInfo<Block> {
+		(self as &SubstrateClient<B, E, Block, RA>).info()
+	}
+
+	fn block_status(&self, id: &BlockId<Block>) -> Result<BlockStatus, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).block_status(id)
+	}
+
+	fn block_hash(&self, block_number: <Block::Header as HeaderT>::Number) -> Result<Option<Block::Hash>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).block_hash(block_number)
+	}
+
+	fn header(&self, id: &BlockId<Block>) -> Result<Option<Block::Header>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).header(id)
+	}
+
+	fn body(&self, id: &BlockId<Block>) -> Result<Option<Vec<Block::Extrinsic>>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).body(id)
+	}
+
+	fn justification(&self, id: &BlockId<Block>) -> Result<Option<Justification>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).justification(id)
+	}
+
+	fn header_proof(&self, block_number: <Block::Header as HeaderT>::Number)
+		-> Result<(Block::Header, StorageProof), Error>
+	{
+		(self as &SubstrateClient<B, E, Block, RA>).header_proof(&BlockId::Number(block_number))
+	}
+
+	fn read_proof(&self, block: &Block::Hash, keys: &[Vec<u8>]) -> Result<StorageProof, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).read_proof(&BlockId::Hash(block.clone()), keys)
+	}
+
+	fn read_child_proof(
+		&self,
+		block: &Block::Hash,
+		storage_key: &[u8],
+		keys: &[Vec<u8>],
+	) -> Result<StorageProof, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>)
+			.read_child_proof(&BlockId::Hash(block.clone()), storage_key, keys)
+	}
+
+	fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec<u8>, StorageProof), Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).execution_proof(&BlockId::Hash(block.clone()), method, data)
+	}
+
+	fn key_changes_proof(
+		&self,
+		first: Block::Hash,
+		last: Block::Hash,
+		min: Block::Hash,
+		max: Block::Hash,
+		storage_key: Option<&StorageKey>,
+		key: &StorageKey,
+	) -> Result<ChangesProof<Block::Header>, Error> {
+		(self as &SubstrateClient<B, E, Block, RA>).key_changes_proof(first, last, min, max, storage_key, key)
+	}
+
+	fn is_descendent_of(&self, base: &Block::Hash, block: &Block::Hash) -> Result<bool, Error> {
+		if base == block {
+			return Ok(false);
+		}
+
+		let ancestor = header_metadata::lowest_common_ancestor(self, *block, *base)?;
+
+		Ok(ancestor.hash == *base)
+	}
+}

substrate/client/network/src/config.rs

@@ -0,0 +1,539 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Configuration of the networking layer.
+//!
+//! The [`Params`] struct is the struct that must be passed in order to initialize the networking.
+//! See the documentation of [`Params`].
+
+pub use crate::protocol::ProtocolConfig;
+pub use libp2p::{identity, core::PublicKey, wasm_ext::ExtTransport, build_multiaddr};
+
+use crate::chain::{Client, FinalityProofProvider};
+use crate::on_demand_layer::OnDemand;
+use crate::service::{ExHashT, TransactionPool};
+use bitflags::bitflags;
+use consensus::{block_validation::BlockAnnounceValidator, import_queue::ImportQueue};
+use sr_primitives::traits::{Block as BlockT};
+use libp2p::identity::{Keypair, ed25519};
+use libp2p::wasm_ext;
+use libp2p::{PeerId, Multiaddr, multiaddr};
+use core::{fmt, iter};
+use std::{error::Error, fs, io::{self, Write}, net::Ipv4Addr, path::{Path, PathBuf}, sync::Arc};
+use zeroize::Zeroize;
+
+/// Network initialization parameters.
+pub struct Params<B: BlockT, S, H: ExHashT> {
+	/// Assigned roles for our node (full, light, ...).
+	pub roles: Roles,
+
+	/// Network layer configuration.
+	pub network_config: NetworkConfiguration,
+
+	/// Client that contains the blockchain.
+	pub chain: Arc<dyn Client<B>>,
+
+	/// Finality proof provider.
+	///
+	/// This object, if `Some`, is used when a node on the network requests a proof of finality
+	/// from us.
+	pub finality_proof_provider: Option<Arc<dyn FinalityProofProvider<B>>>,
+
+	/// How to build requests for proofs of finality.
+	///
+	/// This object, if `Some`, is used when we need a proof of finality from another node.
+	pub finality_proof_request_builder: Option<BoxFinalityProofRequestBuilder<B>>,
+
+	/// The `OnDemand` object acts as a "receiver" for block data requests from the client.
+	/// If `Some`, the network worker will process these requests and answer them.
+	/// Normally used only for light clients.
+	pub on_demand: Option<Arc<OnDemand<B>>>,
+
+	/// Pool of transactions.
+	///
+	/// The network worker will fetch transactions from this object in order to propagate them on
+	/// the network.
+	pub transaction_pool: Arc<dyn TransactionPool<H, B>>,
+
+	/// Name of the protocol to use on the wire. Should be different for each chain.
+	pub protocol_id: ProtocolId,
+
+	/// Import queue to use.
+	///
+	/// The import queue is the component that verifies that blocks received from other nodes are
+	/// valid.
+	pub import_queue: Box<dyn ImportQueue<B>>,
+
+	/// Customization of the network. Use this to plug additional networking capabilities.
+	pub specialization: S,
+
+	/// Type to check incoming block announcements.
+	pub block_announce_validator: Box<dyn BlockAnnounceValidator<B> + Send>,
+}
+
+bitflags! {
+	/// Bitmask of the roles that a node fulfills.
+	pub struct Roles: u8 {
+		/// No network.
+		const NONE = 0b00000000;
+		/// Full node, does not participate in consensus.
+		const FULL = 0b00000001;
+		/// Light client node.
+		const LIGHT = 0b00000010;
+		/// Act as an authority
+		const AUTHORITY = 0b00000100;
+	}
+}
+
+impl Roles {
+	/// Does this role represents a client that holds full chain data locally?
+	pub fn is_full(&self) -> bool {
+		self.intersects(Roles::FULL | Roles::AUTHORITY)
+	}
+
+	/// Does this role represents a client that does not participates in the consensus?
+	pub fn is_authority(&self) -> bool {
+		*self == Roles::AUTHORITY
+	}
+
+	/// Does this role represents a client that does not hold full chain data locally?
+	pub fn is_light(&self) -> bool {
+		!self.is_full()
+	}
+}
+
+impl codec::Encode for Roles {
+	fn encode_to<T: codec::Output>(&self, dest: &mut T) {
+		dest.push_byte(self.bits())
+	}
+}
+
+impl codec::EncodeLike for Roles {}
+
+impl codec::Decode for Roles {
+	fn decode<I: codec::Input>(input: &mut I) -> Result<Self, codec::Error> {
+		Self::from_bits(input.read_byte()?).ok_or_else(|| codec::Error::from("Invalid bytes"))
+	}
+}
+
+/// Finality proof request builder.
+pub trait FinalityProofRequestBuilder<B: BlockT>: Send {
+	/// Build data blob, associated with the request.
+	fn build_request_data(&mut self, hash: &B::Hash) -> Vec<u8>;
+}
+
+/// Implementation of `FinalityProofRequestBuilder` that builds a dummy empty request.
+#[derive(Debug, Default)]
+pub struct DummyFinalityProofRequestBuilder;
+
+impl<B: BlockT> FinalityProofRequestBuilder<B> for DummyFinalityProofRequestBuilder {
+	fn build_request_data(&mut self, _: &B::Hash) -> Vec<u8> {
+		Vec::new()
+	}
+}
+
+/// Shared finality proof request builder struct used by the queue.
+pub type BoxFinalityProofRequestBuilder<B> = Box<dyn FinalityProofRequestBuilder<B> + Send + Sync>;
+
+/// Name of a protocol, transmitted on the wire. Should be unique for each chain.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct ProtocolId(smallvec::SmallVec<[u8; 6]>);
+
+impl<'a> From<&'a [u8]> for ProtocolId {
+	fn from(bytes: &'a [u8]) -> ProtocolId {
+		ProtocolId(bytes.into())
+	}
+}
+
+impl ProtocolId {
+	/// Exposes the `ProtocolId` as bytes.
+	pub fn as_bytes(&self) -> &[u8] {
+		self.0.as_ref()
+	}
+}
+
+/// Parses a string address and splits it into Multiaddress and PeerId, if
+/// valid.
+///
+/// # Example
+///
+/// ```
+/// # use substrate_network::{Multiaddr, PeerId, config::parse_str_addr};
+/// let (peer_id, addr) = parse_str_addr(
+/// 	"/ip4/198.51.100.19/tcp/30333/p2p/QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV"
+/// ).unwrap();
+/// assert_eq!(peer_id, "QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV".parse::<PeerId>().unwrap());
+/// assert_eq!(addr, "/ip4/198.51.100.19/tcp/30333".parse::<Multiaddr>().unwrap());
+/// ```
+///
+pub fn parse_str_addr(addr_str: &str) -> Result<(PeerId, Multiaddr), ParseErr> {
+	let addr: Multiaddr = addr_str.parse()?;
+	parse_addr(addr)
+}
+
+/// Splits a Multiaddress into a Multiaddress and PeerId.
+pub fn parse_addr(mut addr: Multiaddr)-> Result<(PeerId, Multiaddr), ParseErr> {
+	let who = match addr.pop() {
+		Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key)
+			.map_err(|_| ParseErr::InvalidPeerId)?,
+		_ => return Err(ParseErr::PeerIdMissing),
+	};
+
+	Ok((who, addr))
+}
+
+/// Error that can be generated by `parse_str_addr`.
+#[derive(Debug)]
+pub enum ParseErr {
+	/// Error while parsing the multiaddress.
+	MultiaddrParse(multiaddr::Error),
+	/// Multihash of the peer ID is invalid.
+	InvalidPeerId,
+	/// The peer ID is missing from the address.
+	PeerIdMissing,
+}
+
+impl fmt::Display for ParseErr {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		match self {
+			ParseErr::MultiaddrParse(err) => write!(f, "{}", err),
+			ParseErr::InvalidPeerId => write!(f, "Peer id at the end of the address is invalid"),
+			ParseErr::PeerIdMissing => write!(f, "Peer id is missing from the address"),
+		}
+	}
+}
+
+impl std::error::Error for ParseErr {
+	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+		match self {
+			ParseErr::MultiaddrParse(err) => Some(err),
+			ParseErr::InvalidPeerId => None,
+			ParseErr::PeerIdMissing => None,
+		}
+	}
+}
+
+impl From<multiaddr::Error> for ParseErr {
+	fn from(err: multiaddr::Error) -> ParseErr {
+		ParseErr::MultiaddrParse(err)
+	}
+}
+
+/// Network service configuration.
+#[derive(Clone, Debug)]
+pub struct NetworkConfiguration {
+	/// Directory path to store general network configuration. None means nothing will be saved.
+	pub config_path: Option<String>,
+	/// Directory path to store network-specific configuration. None means nothing will be saved.
+	pub net_config_path: Option<String>,
+	/// Multiaddresses to listen for incoming connections.
+	pub listen_addresses: Vec<Multiaddr>,
+	/// Multiaddresses to advertise. Detected automatically if empty.
+	pub public_addresses: Vec<Multiaddr>,
+	/// List of initial node addresses
+	pub boot_nodes: Vec<String>,
+	/// The node key configuration, which determines the node's network identity keypair.
+	pub node_key: NodeKeyConfig,
+	/// Maximum allowed number of incoming connections.
+	pub in_peers: u32,
+	/// Number of outgoing connections we're trying to maintain.
+	pub out_peers: u32,
+	/// List of reserved node addresses.
+	pub reserved_nodes: Vec<String>,
+	/// The non-reserved peer mode.
+	pub non_reserved_mode: NonReservedPeerMode,
+	/// Client identifier. Sent over the wire for debugging purposes.
+	pub client_version: String,
+	/// Name of the node. Sent over the wire for debugging purposes.
+	pub node_name: String,
+	/// Configuration for the transport layer.
+	pub transport: TransportConfig,
+	/// Maximum number of peers to ask the same blocks in parallel.
+	pub max_parallel_downloads: u32,
+}
+
+impl Default for NetworkConfiguration {
+	fn default() -> Self {
+		NetworkConfiguration {
+			config_path: None,
+			net_config_path: None,
+			listen_addresses: Vec::new(),
+			public_addresses: Vec::new(),
+			boot_nodes: Vec::new(),
+			node_key: NodeKeyConfig::Ed25519(Secret::New),
+			in_peers: 25,
+			out_peers: 75,
+			reserved_nodes: Vec::new(),
+			non_reserved_mode: NonReservedPeerMode::Accept,
+			client_version: "unknown".into(),
+			node_name: "unknown".into(),
+			transport: TransportConfig::Normal {
+				enable_mdns: false,
+				allow_private_ipv4: true,
+				wasm_external_transport: None,
+			},
+			max_parallel_downloads: 5,
+		}
+	}
+}
+
+impl NetworkConfiguration {
+	/// Create a new instance of default settings.
+	pub fn new() -> Self {
+		Self::default()
+	}
+
+	/// Create new default configuration for localhost-only connection with random port (useful for testing)
+	pub fn new_local() -> NetworkConfiguration {
+		let mut config = NetworkConfiguration::new();
+		config.listen_addresses = vec![
+			iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
+				.chain(iter::once(multiaddr::Protocol::Tcp(0)))
+				.collect()
+		];
+		config
+	}
+
+	/// Create new default configuration for localhost-only connection with random port (useful for testing)
+	pub fn new_memory() -> NetworkConfiguration {
+		let mut config = NetworkConfiguration::new();
+		config.listen_addresses = vec![
+			iter::once(multiaddr::Protocol::Ip4(Ipv4Addr::new(127, 0, 0, 1)))
+				.chain(iter::once(multiaddr::Protocol::Tcp(0)))
+				.collect()
+		];
+		config
+	}
+}
+
+/// Configuration for the transport layer.
+#[derive(Clone, Debug)]
+pub enum TransportConfig {
+	/// Normal transport mode.
+	Normal {
+		/// If true, the network will use mDNS to discover other libp2p nodes on the local network
+		/// and connect to them if they support the same chain.
+		enable_mdns: bool,
+
+		/// If true, allow connecting to private IPv4 addresses (as defined in
+		/// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address has been passed in
+		/// [`NetworkConfiguration::reserved_nodes`] or [`NetworkConfiguration::boot_nodes`].
+		allow_private_ipv4: bool,
+
+		/// Optional external implementation of a libp2p transport. Used in WASM contexts where we
+		/// need some binding between the networking provided by the operating system or environment
+		/// and libp2p.
+		///
+		/// This parameter exists whatever the target platform is, but it is expected to be set to
+		/// `Some` only when compiling for WASM.
+		wasm_external_transport: Option<wasm_ext::ExtTransport>,
+	},
+
+	/// Only allow connections within the same process.
+	/// Only addresses of the form `/memory/...` will be supported.
+	MemoryOnly,
+}
+
+/// The policy for connections to non-reserved peers.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum NonReservedPeerMode {
+	/// Accept them. This is the default.
+	Accept,
+	/// Deny them.
+	Deny,
+}
+
+impl NonReservedPeerMode {
+	/// Attempt to parse the peer mode from a string.
+	pub fn parse(s: &str) -> Option<Self> {
+		match s {
+			"accept" => Some(NonReservedPeerMode::Accept),
+			"deny" => Some(NonReservedPeerMode::Deny),
+			_ => None,
+		}
+	}
+}
+
+/// The configuration of a node's secret key, describing the type of key
+/// and how it is obtained. A node's identity keypair is the result of
+/// the evaluation of the node key configuration.
+#[derive(Clone, Debug)]
+pub enum NodeKeyConfig {
+	/// A Ed25519 secret key configuration.
+	Ed25519(Secret<ed25519::SecretKey>)
+}
+
+/// The options for obtaining a Ed25519 secret key.
+pub type Ed25519Secret = Secret<ed25519::SecretKey>;
+
+/// The configuration options for obtaining a secret key `K`.
+#[derive(Clone)]
+pub enum Secret<K> {
+	/// Use the given secret key `K`.
+	Input(K),
+	/// Read the secret key from a file. If the file does not exist,
+	/// it is created with a newly generated secret key `K`. The format
+	/// of the file is determined by `K`:
+	///
+	///   * `ed25519::SecretKey`: An unencoded 32 bytes Ed25519 secret key.
+	File(PathBuf),
+	/// Always generate a new secret key `K`.
+	New
+}
+
+impl<K> fmt::Debug for Secret<K> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			Secret::Input(_) => f.debug_tuple("Secret::Input").finish(),
+			Secret::File(path) => f.debug_tuple("Secret::File").field(path).finish(),
+			Secret::New => f.debug_tuple("Secret::New").finish(),
+		}
+	}
+}
+
+impl NodeKeyConfig {
+	/// Evaluate a `NodeKeyConfig` to obtain an identity `Keypair`:
+	///
+	///  * If the secret is configured as input, the corresponding keypair is returned.
+	///
+	///  * If the secret is configured as a file, it is read from that file, if it exists.
+	///    Otherwise a new secret is generated and stored. In either case, the
+	///    keypair obtained from the secret is returned.
+	///
+	///  * If the secret is configured to be new, it is generated and the corresponding
+	///    keypair is returned.
+	pub fn into_keypair(self) -> io::Result<Keypair> {
+		use NodeKeyConfig::*;
+		match self {
+			Ed25519(Secret::New) =>
+				Ok(Keypair::generate_ed25519()),
+
+			Ed25519(Secret::Input(k)) =>
+				Ok(Keypair::Ed25519(k.into())),
+
+			Ed25519(Secret::File(f)) =>
+				get_secret(f,
+					|mut b| ed25519::SecretKey::from_bytes(&mut b),
+					ed25519::SecretKey::generate,
+					|b| b.as_ref().to_vec())
+				.map(ed25519::Keypair::from)
+				.map(Keypair::Ed25519),
+		}
+	}
+}
+
+/// Load a secret key from a file, if it exists, or generate a
+/// new secret key and write it to that file. In either case,
+/// the secret key is returned.
+fn get_secret<P, F, G, E, W, K>(file: P, parse: F, generate: G, serialize: W) -> io::Result<K>
+where
+	P: AsRef<Path>,
+	F: for<'r> FnOnce(&'r mut [u8]) -> Result<K, E>,
+	G: FnOnce() -> K,
+	E: Error + Send + Sync + 'static,
+	W: Fn(&K) -> Vec<u8>,
+{
+	std::fs::read(&file)
+		.and_then(|mut sk_bytes|
+			parse(&mut sk_bytes)
+				.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)))
+		.or_else(|e| {
+			if e.kind() == io::ErrorKind::NotFound {
+				file.as_ref().parent().map_or(Ok(()), fs::create_dir_all)?;
+				let sk = generate();
+				let mut sk_vec = serialize(&sk);
+				write_secret_file(file, &sk_vec)?;
+				sk_vec.zeroize();
+				Ok(sk)
+			} else {
+				Err(e)
+			}
+		})
+}
+
+/// Write secret bytes to a file.
+fn write_secret_file<P>(path: P, sk_bytes: &[u8]) -> io::Result<()>
+where
+	P: AsRef<Path>
+{
+	let mut file = open_secret_file(&path)?;
+	file.write_all(sk_bytes)
+}
+
+/// Opens a file containing a secret key in write mode.
+#[cfg(unix)]
+fn open_secret_file<P>(path: P) -> io::Result<fs::File>
+where
+	P: AsRef<Path>
+{
+	use std::os::unix::fs::OpenOptionsExt;
+	fs::OpenOptions::new()
+		.write(true)
+		.create_new(true)
+		.mode(0o600)
+		.open(path)
+}
+
+/// Opens a file containing a secret key in write mode.
+#[cfg(not(unix))]
+fn open_secret_file<P>(path: P) -> Result<fs::File, io::Error>
+where
+	P: AsRef<Path>
+{
+	fs::OpenOptions::new()
+		.write(true)
+		.create_new(true)
+		.open(path)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use tempdir::TempDir;
+
+	fn secret_bytes(kp: &Keypair) -> Vec<u8> {
+		match kp {
+			Keypair::Ed25519(p) => p.secret().as_ref().iter().cloned().collect(),
+			Keypair::Secp256k1(p) => p.secret().to_bytes().to_vec(),
+			_ => panic!("Unexpected keypair.")
+		}
+	}
+
+	#[test]
+	fn test_secret_file() {
+		let tmp = TempDir::new("x").unwrap();
+		std::fs::remove_dir(tmp.path()).unwrap(); // should be recreated
+		let file = tmp.path().join("x").to_path_buf();
+		let kp1 = NodeKeyConfig::Ed25519(Secret::File(file.clone())).into_keypair().unwrap();
+		let kp2 = NodeKeyConfig::Ed25519(Secret::File(file.clone())).into_keypair().unwrap();
+		assert!(file.is_file() && secret_bytes(&kp1) == secret_bytes(&kp2))
+	}
+
+	#[test]
+	fn test_secret_input() {
+		let sk = ed25519::SecretKey::generate();
+		let kp1 = NodeKeyConfig::Ed25519(Secret::Input(sk.clone())).into_keypair().unwrap();
+		let kp2 = NodeKeyConfig::Ed25519(Secret::Input(sk)).into_keypair().unwrap();
+		assert!(secret_bytes(&kp1) == secret_bytes(&kp2));
+	}
+
+	#[test]
+	fn test_secret_new() {
+		let kp1 = NodeKeyConfig::Ed25519(Secret::New).into_keypair().unwrap();
+		let kp2 = NodeKeyConfig::Ed25519(Secret::New).into_keypair().unwrap();
+		assert!(secret_bytes(&kp1) != secret_bytes(&kp2));
+	}
+}

substrate/client/network/src/debug_info.rs

@@ -0,0 +1,322 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use fnv::FnvHashMap;
+use futures::prelude::*;
+use futures03::{StreamExt as _, TryStreamExt as _};
+use libp2p::Multiaddr;
+use libp2p::core::{ConnectedPoint, either::EitherOutput, PeerId, PublicKey};
+use libp2p::swarm::{IntoProtocolsHandler, IntoProtocolsHandlerSelect, ProtocolsHandler};
+use libp2p::swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters};
+use libp2p::identify::{Identify, IdentifyEvent, IdentifyInfo};
+use libp2p::ping::{Ping, PingConfig, PingEvent, PingSuccess};
+use log::{debug, trace, error};
+use std::collections::hash_map::Entry;
+use std::time::{Duration, Instant};
+use tokio_io::{AsyncRead, AsyncWrite};
+use futures_timer::Interval;
+
+/// Time after we disconnect from a node before we purge its information from the cache.
+const CACHE_EXPIRE: Duration = Duration::from_secs(10 * 60);
+/// Interval at which we perform garbage collection on the node info.
+const GARBAGE_COLLECT_INTERVAL: Duration = Duration::from_secs(2 * 60);
+
+/// Implementation of `NetworkBehaviour` that holds information about nodes in cache for diagnostic
+/// purposes.
+pub struct DebugInfoBehaviour<TSubstream> {
+	/// Periodically ping nodes, and close the connection if it's unresponsive.
+	ping: Ping<TSubstream>,
+	/// Periodically identifies the remote and responds to incoming requests.
+	identify: Identify<TSubstream>,
+	/// Information that we know about all nodes.
+	nodes_info: FnvHashMap<PeerId, NodeInfo>,
+	/// Interval at which we perform garbage collection in `nodes_info`.
+	garbage_collect: Box<dyn Stream<Item = (), Error = ()> + Send>,
+}
+
+/// Information about a node we're connected to.
+#[derive(Debug)]
+struct NodeInfo {
+	/// When we will remove the entry about this node from the list, or `None` if we're connected
+	/// to the node.
+	info_expire: Option<Instant>,
+	/// How we're connected to the node.
+	endpoint: ConnectedPoint,
+	/// Version reported by the remote, or `None` if unknown.
+	client_version: Option<String>,
+	/// Latest ping time with this node.
+	latest_ping: Option<Duration>,
+}
+
+impl<TSubstream> DebugInfoBehaviour<TSubstream> {
+	/// Builds a new `DebugInfoBehaviour`.
+	pub fn new(
+		user_agent: String,
+		local_public_key: PublicKey,
+	) -> Self {
+		let identify = {
+			let proto_version = "/substrate/1.0".to_string();
+			Identify::new(proto_version, user_agent, local_public_key.clone())
+		};
+
+		DebugInfoBehaviour {
+			ping: Ping::new(PingConfig::new()),
+			identify,
+			nodes_info: FnvHashMap::default(),
+			garbage_collect: Box::new(Interval::new(GARBAGE_COLLECT_INTERVAL).map(|()| Ok(())).compat()),
+		}
+	}
+
+	/// Borrows `self` and returns a struct giving access to the information about a node.
+	///
+	/// Returns `None` if we don't know anything about this node. Always returns `Some` for nodes
+	/// we're connected to, meaning that if `None` is returned then we're not connected to that
+	/// node.
+	pub fn node(&self, peer_id: &PeerId) -> Option<Node> {
+		self.nodes_info.get(peer_id).map(Node)
+	}
+
+	/// Inserts a ping time in the cache. Has no effect if we don't have any entry for that node,
+	/// which shouldn't happen.
+	fn handle_ping_report(&mut self, peer_id: &PeerId, ping_time: Duration) {
+		trace!(target: "sub-libp2p", "Ping time with {:?}: {:?}", peer_id, ping_time);
+		if let Some(entry) = self.nodes_info.get_mut(peer_id) {
+			entry.latest_ping = Some(ping_time);
+		} else {
+			error!(target: "sub-libp2p",
+				"Received ping from node we're not connected to {:?}", peer_id);
+		}
+	}
+
+	/// Inserts an identify record in the cache. Has no effect if we don't have any entry for that
+	/// node, which shouldn't happen.
+	fn handle_identify_report(&mut self, peer_id: &PeerId, info: &IdentifyInfo) {
+		trace!(target: "sub-libp2p", "Identified {:?} => {:?}", peer_id, info);
+		if let Some(entry) = self.nodes_info.get_mut(peer_id) {
+			entry.client_version = Some(info.agent_version.clone());
+		} else {
+			error!(target: "sub-libp2p",
+				"Received pong from node we're not connected to {:?}", peer_id);
+		}
+	}
+}
+
+/// Gives access to the information about a node.
+pub struct Node<'a>(&'a NodeInfo);
+
+impl<'a> Node<'a> {
+	/// Returns the endpoint we are connected to or were last connected to.
+	pub fn endpoint(&self) -> &'a ConnectedPoint {
+		&self.0.endpoint
+	}
+
+	/// Returns the latest version information we know of.
+	pub fn client_version(&self) -> Option<&'a str> {
+		self.0.client_version.as_ref().map(|s| &s[..])
+	}
+
+	/// Returns the latest ping time we know of for this node. `None` if we never successfully
+	/// pinged this node.
+	pub fn latest_ping(&self) -> Option<Duration> {
+		self.0.latest_ping
+	}
+}
+
+/// Event that can be emitted by the behaviour.
+#[derive(Debug)]
+pub enum DebugInfoEvent {
+	/// We have obtained debug information from a peer, including the addresses it is listening
+	/// on.
+	Identified {
+		/// Id of the peer that has been identified.
+		peer_id: PeerId,
+		/// Information about the peer.
+		info: IdentifyInfo,
+	},
+}
+
+impl<TSubstream> NetworkBehaviour for DebugInfoBehaviour<TSubstream>
+where TSubstream: AsyncRead + AsyncWrite {
+	type ProtocolsHandler = IntoProtocolsHandlerSelect<
+		<Ping<TSubstream> as NetworkBehaviour>::ProtocolsHandler,
+		<Identify<TSubstream> as NetworkBehaviour>::ProtocolsHandler
+	>;
+	type OutEvent = DebugInfoEvent;
+
+	fn new_handler(&mut self) -> Self::ProtocolsHandler {
+		IntoProtocolsHandler::select(self.ping.new_handler(), self.identify.new_handler())
+	}
+
+	fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
+		let mut list = self.ping.addresses_of_peer(peer_id);
+		list.extend_from_slice(&self.identify.addresses_of_peer(peer_id));
+		list
+	}
+
+	fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) {
+		self.ping.inject_connected(peer_id.clone(), endpoint.clone());
+		self.identify.inject_connected(peer_id.clone(), endpoint.clone());
+
+		match self.nodes_info.entry(peer_id) {
+			Entry::Vacant(e) => {
+				e.insert(NodeInfo {
+					info_expire: None,
+					endpoint,
+					client_version: None,
+					latest_ping: None,
+				});
+			}
+			Entry::Occupied(e) => {
+				let e = e.into_mut();
+				if e.info_expire.as_ref().map(|exp| *exp < Instant::now()).unwrap_or(false) {
+					e.client_version = None;
+					e.latest_ping = None;
+				}
+				e.info_expire = None;
+				e.endpoint = endpoint;
+			}
+		}
+	}
+
+	fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) {
+		self.ping.inject_disconnected(peer_id, endpoint.clone());
+		self.identify.inject_disconnected(peer_id, endpoint);
+
+		if let Some(entry) = self.nodes_info.get_mut(peer_id) {
+			entry.info_expire = Some(Instant::now() + CACHE_EXPIRE);
+		} else {
+			error!(target: "sub-libp2p",
+				"Disconnected from node we were not connected to {:?}", peer_id);
+		}
+	}
+
+	fn inject_node_event(
+		&mut self,
+		peer_id: PeerId,
+		event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent
+	) {
+		match event {
+			EitherOutput::First(event) => self.ping.inject_node_event(peer_id, event),
+			EitherOutput::Second(event) => self.identify.inject_node_event(peer_id, event),
+		}
+	}
+
+	fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) {
+		self.ping.inject_replaced(peer_id.clone(), closed_endpoint.clone(), new_endpoint.clone());
+		self.identify.inject_replaced(peer_id.clone(), closed_endpoint, new_endpoint.clone());
+
+		if let Some(entry) = self.nodes_info.get_mut(&peer_id) {
+			entry.endpoint = new_endpoint;
+		} else {
+			error!(target: "sub-libp2p",
+				"Disconnected from node we were not connected to {:?}", peer_id);
+		}
+	}
+
+	fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) {
+		self.ping.inject_addr_reach_failure(peer_id, addr, error);
+		self.identify.inject_addr_reach_failure(peer_id, addr, error);
+	}
+
+	fn inject_dial_failure(&mut self, peer_id: &PeerId) {
+		self.ping.inject_dial_failure(peer_id);
+		self.identify.inject_dial_failure(peer_id);
+	}
+
+	fn inject_new_listen_addr(&mut self, addr: &Multiaddr) {
+		self.ping.inject_new_listen_addr(addr);
+		self.identify.inject_new_listen_addr(addr);
+	}
+
+	fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) {
+		self.ping.inject_expired_listen_addr(addr);
+		self.identify.inject_expired_listen_addr(addr);
+	}
+
+	fn inject_new_external_addr(&mut self, addr: &Multiaddr) {
+		self.ping.inject_new_external_addr(addr);
+		self.identify.inject_new_external_addr(addr);
+	}
+
+	fn poll(
+		&mut self,
+		params: &mut impl PollParameters
+	) -> Async<
+		NetworkBehaviourAction<
+			<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
+			Self::OutEvent
+		>
+	> {
+		loop {
+			match self.ping.poll(params) {
+				Async::NotReady => break,
+				Async::Ready(NetworkBehaviourAction::GenerateEvent(ev)) => {
+					if let PingEvent { peer, result: Ok(PingSuccess::Ping { rtt }) } = ev {
+						self.handle_ping_report(&peer, rtt)
+					}
+				},
+				Async::Ready(NetworkBehaviourAction::DialAddress { address }) =>
+					return Async::Ready(NetworkBehaviourAction::DialAddress { address }),
+				Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }) =>
+					return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }),
+				Async::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) =>
+					return Async::Ready(NetworkBehaviourAction::SendEvent {
+						peer_id,
+						event: EitherOutput::First(event)
+					}),
+				Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) =>
+					return Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }),
+			}
+		}
+
+		loop {
+			match self.identify.poll(params) {
+				Async::NotReady => break,
+				Async::Ready(NetworkBehaviourAction::GenerateEvent(event)) => {
+					match event {
+						IdentifyEvent::Received { peer_id, info, .. } => {
+							self.handle_identify_report(&peer_id, &info);
+							let event = DebugInfoEvent::Identified { peer_id, info };
+							return Async::Ready(NetworkBehaviourAction::GenerateEvent(event));
+						}
+						IdentifyEvent::Error { peer_id, error } =>
+							debug!(target: "sub-libp2p", "Identification with peer {:?} failed => {}", peer_id, error),
+						IdentifyEvent::Sent { .. } => {}
+					}
+				},
+				Async::Ready(NetworkBehaviourAction::DialAddress { address }) =>
+					return Async::Ready(NetworkBehaviourAction::DialAddress { address }),
+				Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }) =>
+					return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }),
+				Async::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) =>
+					return Async::Ready(NetworkBehaviourAction::SendEvent {
+						peer_id,
+						event: EitherOutput::Second(event)
+					}),
+				Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) =>
+					return Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }),
+			}
+		}
+
+		while let Ok(Async::Ready(Some(_))) = self.garbage_collect.poll() {
+			self.nodes_info.retain(|_, node| {
+				node.info_expire.as_ref().map(|exp| *exp >= Instant::now()).unwrap_or(true)
+			});
+		}
+
+		Async::NotReady
+	}
+}

substrate/client/network/src/discovery.rs

@@ -0,0 +1,541 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Discovery mechanisms of Substrate.
+//!
+//! The `DiscoveryBehaviour` struct implements the `NetworkBehaviour` trait of libp2p and is
+//! responsible for discovering other nodes that are part of the network.
+//!
+//! Substrate uses the following mechanisms in order to discover nodes that are part of the network:
+//!
+//! - Bootstrap nodes. These are hard-coded node identities and addresses passed in the constructor
+//! of the `DiscoveryBehaviour`. You can also call `add_known_address` later to add an entry.
+//!
+//! - mDNS. Discovers nodes on the local network by broadcasting UDP packets.
+//!
+//! - Kademlia random walk. Once connected, we perform random Kademlia `FIND_NODE` requests in
+//! order for nodes to propagate to us their view of the network. This is performed automatically
+//! by the `DiscoveryBehaviour`.
+//!
+//! Additionally, the `DiscoveryBehaviour` is also capable of storing and loading value in the
+//! network-wide DHT.
+//!
+//! ## Usage
+//!
+//! The `DiscoveryBehaviour` generates events of type `DiscoveryOut`, most notably
+//! `DiscoveryOut::Discovered` that is generated whenever we discover a node.
+//! Only the identity of the node is returned. The node's addresses are stored within the
+//! `DiscoveryBehaviour` and can be queried through the `NetworkBehaviour` trait.
+//!
+//! **Important**: In order for the discovery mechanism to work properly, there needs to be an
+//! active mechanism that asks nodes for the addresses they are listening on. Whenever we learn
+//! of a node's address, you must call `add_self_reported_address`.
+//!
+
+use futures::prelude::*;
+use futures_timer::Delay;
+use futures03::{compat::Compat, TryFutureExt as _};
+use libp2p::core::{ConnectedPoint, Multiaddr, PeerId, PublicKey};
+use libp2p::swarm::{ProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters};
+use libp2p::kad::{Kademlia, KademliaEvent, Quorum, Record};
+use libp2p::kad::GetClosestPeersError;
+use libp2p::kad::record::{self, store::MemoryStore};
+#[cfg(not(target_os = "unknown"))]
+use libp2p::{swarm::toggle::Toggle};
+#[cfg(not(target_os = "unknown"))]
+use libp2p::core::{nodes::Substream, muxing::StreamMuxerBox};
+#[cfg(not(target_os = "unknown"))]
+use libp2p::mdns::{Mdns, MdnsEvent};
+use libp2p::multiaddr::Protocol;
+use log::{debug, info, trace, warn};
+use std::{cmp, collections::VecDeque, time::Duration};
+use tokio_io::{AsyncRead, AsyncWrite};
+use primitives::hexdisplay::HexDisplay;
+
+/// Implementation of `NetworkBehaviour` that discovers the nodes on the network.
+pub struct DiscoveryBehaviour<TSubstream> {
+	/// User-defined list of nodes and their addresses. Typically includes bootstrap nodes and
+	/// reserved nodes.
+	user_defined: Vec<(PeerId, Multiaddr)>,
+	/// Kademlia requests and answers.
+	kademlia: Kademlia<TSubstream, MemoryStore>,
+	/// Discovers nodes on the local network.
+	#[cfg(not(target_os = "unknown"))]
+	mdns: Toggle<Mdns<Substream<StreamMuxerBox>>>,
+	/// Stream that fires when we need to perform the next random Kademlia query.
+	next_kad_random_query: Compat<Delay>,
+	/// After `next_kad_random_query` triggers, the next one triggers after this duration.
+	duration_to_next_kad: Duration,
+	/// Discovered nodes to return.
+	discoveries: VecDeque<PeerId>,
+	/// Identity of our local node.
+	local_peer_id: PeerId,
+	/// Number of nodes we're currently connected to.
+	num_connections: u64,
+	/// If false, `addresses_of_peer` won't return any private IPv4 address, except for the ones
+	/// stored in `user_defined`.
+	allow_private_ipv4: bool,
+}
+
+impl<TSubstream> DiscoveryBehaviour<TSubstream> {
+	/// Builds a new `DiscoveryBehaviour`.
+	///
+	/// `user_defined` is a list of known address for nodes that never expire.
+	pub fn new(
+		local_public_key: PublicKey,
+		user_defined: Vec<(PeerId, Multiaddr)>,
+		enable_mdns: bool,
+		allow_private_ipv4: bool,
+	) -> Self {
+		if enable_mdns {
+			#[cfg(target_os = "unknown")]
+			warn!(target: "sub-libp2p", "mDNS is not available on this platform");
+		}
+
+		let local_id = local_public_key.clone().into_peer_id();
+		let store = MemoryStore::new(local_id.clone());
+		let mut kademlia = Kademlia::new(local_id.clone(), store);
+		for (peer_id, addr) in &user_defined {
+			kademlia.add_address(peer_id, addr.clone());
+		}
+
+		DiscoveryBehaviour {
+			user_defined,
+			kademlia,
+			next_kad_random_query: Delay::new(Duration::new(0, 0)).compat(),
+			duration_to_next_kad: Duration::from_secs(1),
+			discoveries: VecDeque::new(),
+			local_peer_id: local_public_key.into_peer_id(),
+			num_connections: 0,
+			allow_private_ipv4,
+			#[cfg(not(target_os = "unknown"))]
+			mdns: if enable_mdns {
+				match Mdns::new() {
+					Ok(mdns) => Some(mdns).into(),
+					Err(err) => {
+						warn!(target: "sub-libp2p", "Failed to initialize mDNS: {:?}", err);
+						None.into()
+					}
+				}
+			} else {
+				None.into()
+			},
+		}
+	}
+
+	/// Returns the list of nodes that we know exist in the network.
+	pub fn known_peers(&mut self) -> impl Iterator<Item = &PeerId> {
+		self.kademlia.kbuckets_entries()
+	}
+
+	/// Adds a hard-coded address for the given peer, that never expires.
+	///
+	/// This adds an entry to the parameter that was passed to `new`.
+	///
+	/// If we didn't know this address before, also generates a `Discovered` event.
+	pub fn add_known_address(&mut self, peer_id: PeerId, addr: Multiaddr) {
+		if self.user_defined.iter().all(|(p, a)| *p != peer_id && *a != addr) {
+			self.discoveries.push_back(peer_id.clone());
+			self.user_defined.push((peer_id, addr));
+		}
+	}
+
+	/// Call this method when a node reports an address for itself.
+	///
+	/// **Note**: It is important that you call this method, otherwise the discovery mechanism will
+	/// not properly work.
+	pub fn add_self_reported_address(&mut self, peer_id: &PeerId, addr: Multiaddr) {
+		self.kademlia.add_address(peer_id, addr);
+	}
+
+	/// Start fetching a record from the DHT.
+	///
+	/// A corresponding `ValueFound` or `ValueNotFound` event will later be generated.
+	pub fn get_value(&mut self, key: &record::Key) {
+		self.kademlia.get_record(key, Quorum::One)
+	}
+
+	/// Start putting a record into the DHT. Other nodes can later fetch that value with
+	/// `get_value`.
+	///
+	/// A corresponding `ValuePut` or `ValuePutFailed` event will later be generated.
+	pub fn put_value(&mut self, key: record::Key, value: Vec<u8>) {
+		self.kademlia.put_record(Record::new(key, value), Quorum::All);
+	}
+}
+
+/// Event generated by the `DiscoveryBehaviour`.
+pub enum DiscoveryOut {
+	/// The address of a peer has been added to the Kademlia routing table.
+	///
+	/// Can be called multiple times with the same identity.
+	Discovered(PeerId),
+
+	/// A peer connected to this node for whom no listen address is known.
+	///
+	/// In order for the peer to be added to the Kademlia routing table, a known
+	/// listen address must be added via [`DiscoveryBehaviour::add_self_reported_address`],
+	/// e.g. obtained through the `identify` protocol.
+	UnroutablePeer(PeerId),
+
+	/// The DHT yeided results for the record request, grouped in (key, value) pairs.
+	ValueFound(Vec<(record::Key, Vec<u8>)>),
+
+	/// The record requested was not found in the DHT.
+	ValueNotFound(record::Key),
+
+	/// The record with a given key was successfully inserted into the DHT.
+	ValuePut(record::Key),
+
+	/// Inserting a value into the DHT failed.
+	ValuePutFailed(record::Key),
+}
+
+impl<TSubstream> NetworkBehaviour for DiscoveryBehaviour<TSubstream>
+where
+	TSubstream: AsyncRead + AsyncWrite,
+{
+	type ProtocolsHandler = <Kademlia<TSubstream, MemoryStore> as NetworkBehaviour>::ProtocolsHandler;
+	type OutEvent = DiscoveryOut;
+
+	fn new_handler(&mut self) -> Self::ProtocolsHandler {
+		NetworkBehaviour::new_handler(&mut self.kademlia)
+	}
+
+	fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
+		let mut list = self.user_defined.iter()
+			.filter_map(|(p, a)| if p == peer_id { Some(a.clone()) } else { None })
+			.collect::<Vec<_>>();
+
+		{
+			let mut list_to_filter = self.kademlia.addresses_of_peer(peer_id);
+			#[cfg(not(target_os = "unknown"))]
+			list_to_filter.extend(self.mdns.addresses_of_peer(peer_id));
+
+			if !self.allow_private_ipv4 {
+				list_to_filter.retain(|addr| {
+					if let Some(Protocol::Ip4(addr)) = addr.iter().next() {
+						if addr.is_private() {
+							return false;
+						}
+					}
+
+					true
+				});
+			}
+
+			list.extend(list_to_filter);
+		}
+
+		trace!(target: "sub-libp2p", "Addresses of {:?} are {:?}", peer_id, list);
+		if list.is_empty() {
+			if self.kademlia.kbuckets_entries().any(|p| p == peer_id) {
+				debug!(target: "sub-libp2p", "Requested dialing to {:?} (peer in k-buckets), \
+					and no address was found", peer_id);
+			} else {
+				debug!(target: "sub-libp2p", "Requested dialing to {:?} (peer not in k-buckets), \
+					and no address was found", peer_id);
+			}
+		}
+		list
+	}
+
+	fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) {
+		self.num_connections += 1;
+		NetworkBehaviour::inject_connected(&mut self.kademlia, peer_id, endpoint)
+	}
+
+	fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) {
+		self.num_connections -= 1;
+		NetworkBehaviour::inject_disconnected(&mut self.kademlia, peer_id, endpoint)
+	}
+
+	fn inject_replaced(&mut self, peer_id: PeerId, closed: ConnectedPoint, opened: ConnectedPoint) {
+		NetworkBehaviour::inject_replaced(&mut self.kademlia, peer_id, closed, opened)
+	}
+
+	fn inject_node_event(
+		&mut self,
+		peer_id: PeerId,
+		event: <Self::ProtocolsHandler as ProtocolsHandler>::OutEvent,
+	) {
+		NetworkBehaviour::inject_node_event(&mut self.kademlia, peer_id, event)
+	}
+
+	fn inject_new_external_addr(&mut self, addr: &Multiaddr) {
+		let new_addr = addr.clone()
+			.with(Protocol::P2p(self.local_peer_id.clone().into()));
+		info!(target: "sub-libp2p", "Discovered new external address for our node: {}", new_addr);
+	}
+
+	fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) {
+		info!(target: "sub-libp2p", "No longer listening on {}", addr);
+	}
+
+	fn poll(
+		&mut self,
+		params: &mut impl PollParameters,
+	) -> Async<
+		NetworkBehaviourAction<
+			<Self::ProtocolsHandler as ProtocolsHandler>::InEvent,
+			Self::OutEvent,
+		>,
+	> {
+		// Immediately process the content of `discovered`.
+		if let Some(peer_id) = self.discoveries.pop_front() {
+			let ev = DiscoveryOut::Discovered(peer_id);
+			return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+		}
+
+		// Poll the stream that fires when we need to start a random Kademlia query.
+		loop {
+			match self.next_kad_random_query.poll() {
+				Ok(Async::NotReady) => break,
+				Ok(Async::Ready(_)) => {
+					let random_peer_id = PeerId::random();
+					debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \
+						{:?}", random_peer_id);
+
+					self.kademlia.get_closest_peers(random_peer_id);
+
+					// Schedule the next random query with exponentially increasing delay,
+					// capped at 60 seconds.
+					self.next_kad_random_query = Delay::new(self.duration_to_next_kad).compat();
+					self.duration_to_next_kad = cmp::min(self.duration_to_next_kad * 2,
+						Duration::from_secs(60));
+				},
+				Err(err) => {
+					warn!(target: "sub-libp2p", "Kademlia query timer errored: {:?}", err);
+					break
+				}
+			}
+		}
+
+		// Poll Kademlia.
+		loop {
+			match self.kademlia.poll(params) {
+				Async::NotReady => break,
+				Async::Ready(NetworkBehaviourAction::GenerateEvent(ev)) => match ev {
+					KademliaEvent::UnroutablePeer { peer, .. } => {
+						let ev = DiscoveryOut::UnroutablePeer(peer);
+						return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+					}
+					KademliaEvent::RoutingUpdated { peer, .. } => {
+						let ev = DiscoveryOut::Discovered(peer);
+						return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+					}
+					KademliaEvent::GetClosestPeersResult(res) => {
+						match res {
+							Err(GetClosestPeersError::Timeout { key, peers }) => {
+								debug!(target: "sub-libp2p",
+									"Libp2p => Query for {:?} timed out with {} results",
+									HexDisplay::from(&key), peers.len());
+							},
+							Ok(ok) => {
+								trace!(target: "sub-libp2p",
+									"Libp2p => Query for {:?} yielded {:?} results",
+									HexDisplay::from(&ok.key), ok.peers.len());
+								if ok.peers.is_empty() && self.num_connections != 0 {
+									debug!(target: "sub-libp2p", "Libp2p => Random Kademlia query has yielded empty \
+										results");
+								}
+							}
+						}
+					}
+					KademliaEvent::GetRecordResult(res) => {
+						let ev = match res {
+							Ok(ok) => {
+								let results = ok.records
+									.into_iter()
+									.map(|r| (r.key, r.value))
+									.collect();
+
+								DiscoveryOut::ValueFound(results)
+							}
+							Err(e) => {
+								DiscoveryOut::ValueNotFound(e.into_key())
+							}
+						};
+						return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+					}
+					KademliaEvent::PutRecordResult(res) => {
+						let ev = match res {
+							Ok(ok) => DiscoveryOut::ValuePut(ok.key),
+							Err(e) => {
+								DiscoveryOut::ValuePutFailed(e.into_key())
+							}
+						};
+						return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+					}
+					KademliaEvent::RepublishRecordResult(res) => {
+						match res {
+							Ok(ok) => debug!(target: "sub-libp2p",
+								"Libp2p => Record republished: {:?}",
+								ok.key),
+							Err(e) => warn!(target: "sub-libp2p",
+								"Libp2p => Republishing of record {:?} failed with: {:?}",
+								e.key(), e)
+						}
+					}
+					KademliaEvent::Discovered { .. } => {
+						// We are not interested in these events at the moment.
+					}
+					// We never start any other type of query.
+					e => {
+						warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e)
+					}
+				},
+				Async::Ready(NetworkBehaviourAction::DialAddress { address }) =>
+					return Async::Ready(NetworkBehaviourAction::DialAddress { address }),
+				Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }) =>
+					return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }),
+				Async::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }) =>
+					return Async::Ready(NetworkBehaviourAction::SendEvent { peer_id, event }),
+				Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) =>
+					return Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }),
+			}
+		}
+
+		// Poll mDNS.
+		#[cfg(not(target_os = "unknown"))]
+		loop {
+			match self.mdns.poll(params) {
+				Async::NotReady => break,
+				Async::Ready(NetworkBehaviourAction::GenerateEvent(event)) => {
+					match event {
+						MdnsEvent::Discovered(list) => {
+							self.discoveries.extend(list.into_iter().map(|(peer_id, _)| peer_id));
+							if let Some(peer_id) = self.discoveries.pop_front() {
+								let ev = DiscoveryOut::Discovered(peer_id);
+								return Async::Ready(NetworkBehaviourAction::GenerateEvent(ev));
+							}
+						},
+						MdnsEvent::Expired(_) => {}
+					}
+				},
+				Async::Ready(NetworkBehaviourAction::DialAddress { address }) =>
+					return Async::Ready(NetworkBehaviourAction::DialAddress { address }),
+				Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }) =>
+					return Async::Ready(NetworkBehaviourAction::DialPeer { peer_id }),
+				Async::Ready(NetworkBehaviourAction::SendEvent { event, .. }) =>
+					match event {},		// `event` is an enum with no variant
+				Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }) =>
+					return Async::Ready(NetworkBehaviourAction::ReportObservedAddr { address }),
+			}
+		}
+
+		Async::NotReady
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use futures::prelude::*;
+	use libp2p::identity::Keypair;
+	use libp2p::Multiaddr;
+	use libp2p::core::upgrade;
+	use libp2p::core::transport::{Transport, MemoryTransport};
+	use libp2p::core::upgrade::{InboundUpgradeExt, OutboundUpgradeExt};
+	use libp2p::swarm::Swarm;
+	use std::collections::HashSet;
+	use super::{DiscoveryBehaviour, DiscoveryOut};
+
+	#[test]
+	fn discovery_working() {
+		let mut user_defined = Vec::new();
+
+		// Build swarms whose behaviour is `DiscoveryBehaviour`.
+		let mut swarms = (0..25).map(|_| {
+			let keypair = Keypair::generate_ed25519();
+			let keypair2 = keypair.clone();
+
+			let transport = MemoryTransport
+				.and_then(move |out, endpoint| {
+					let secio = libp2p::secio::SecioConfig::new(keypair2);
+					libp2p::core::upgrade::apply(
+						out,
+						secio,
+						endpoint,
+						libp2p::core::upgrade::Version::V1
+					)
+				})
+				.and_then(move |(peer_id, stream), endpoint| {
+					let peer_id2 = peer_id.clone();
+					let upgrade = libp2p::yamux::Config::default()
+						.map_inbound(move |muxer| (peer_id, muxer))
+						.map_outbound(move |muxer| (peer_id2, muxer));
+					upgrade::apply(stream, upgrade, endpoint, libp2p::core::upgrade::Version::V1)
+				});
+
+			let behaviour = DiscoveryBehaviour::new(keypair.public(), user_defined.clone(), false, true);
+			let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id());
+			let listen_addr: Multiaddr = format!("/memory/{}", rand::random::<u64>()).parse().unwrap();
+
+			if user_defined.is_empty() {
+				user_defined.push((keypair.public().into_peer_id(), listen_addr.clone()));
+			}
+
+			Swarm::listen_on(&mut swarm, listen_addr.clone()).unwrap();
+			(swarm, listen_addr)
+		}).collect::<Vec<_>>();
+
+		// Build a `Vec<HashSet<PeerId>>` with the list of nodes remaining to be discovered.
+		let mut to_discover = (0..swarms.len()).map(|n| {
+			(0..swarms.len()).filter(|p| *p != n)
+				.map(|p| Swarm::local_peer_id(&swarms[p].0).clone())
+				.collect::<HashSet<_>>()
+		}).collect::<Vec<_>>();
+
+		let fut = futures::future::poll_fn::<_, (), _>(move || {
+			'polling: loop {
+				for swarm_n in 0..swarms.len() {
+					match swarms[swarm_n].0.poll().unwrap() {
+						Async::Ready(Some(e)) => {
+							match e {
+								DiscoveryOut::UnroutablePeer(other) => {
+									// Call `add_self_reported_address` to simulate identify happening.
+									let addr = swarms.iter().find_map(|(s, a)|
+										if s.local_peer_id == other {
+											Some(a.clone())
+										} else {
+											None
+										})
+										.unwrap();
+									swarms[swarm_n].0.add_self_reported_address(&other, addr);
+								},
+								DiscoveryOut::Discovered(other) => {
+									to_discover[swarm_n].remove(&other);
+								}
+								_ => {}
+							}
+							continue 'polling
+						}
+						_ => {}
+					}
+				}
+				break
+			}
+
+			if to_discover.iter().all(|l| l.is_empty()) {
+				Ok(Async::Ready(()))
+			} else {
+				Ok(Async::NotReady)
+			}
+		});
+
+		tokio::runtime::Runtime::new().unwrap().block_on(fut).unwrap();
+	}
+}

substrate/client/network/src/error.rs

@@ -0,0 +1,67 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate network possible errors.
+
+use client_api;
+
+use libp2p::{PeerId, Multiaddr};
+
+use std::fmt;
+
+/// Result type alias for the network.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Error type for the network.
+#[derive(derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Io error
+	Io(std::io::Error),
+	/// Client error
+	Client(client_api::error::Error),
+	/// The same bootnode (based on address) is registered with two different peer ids.
+	#[display(
+		fmt = "The same bootnode (`{}`) is registered with two different peer ids: `{}` and `{}`",
+		address,
+		first_id,
+		second_id,
+	)]
+	DuplicateBootnode {
+		/// The address of the bootnode.
+		address: Multiaddr,
+		/// The first peer id that was found for the bootnode.
+		first_id: PeerId,
+		/// The second peer id that was found for the bootnode.
+		second_id: PeerId,
+	},
+}
+
+// Make `Debug` use the `Display` implementation.
+impl fmt::Debug for Error {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		fmt::Display::fmt(self, f)
+	}
+}
+
+impl std::error::Error for Error {
+	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+		match self {
+			Error::Io(ref err) => Some(err),
+			Error::Client(ref err) => Some(err),
+			Error::DuplicateBootnode { .. } => None,
+		}
+	}
+}

substrate/client/network/src/legacy_proto/handler.rs

@@ -0,0 +1,651 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::legacy_proto::upgrade::{RegisteredProtocol, RegisteredProtocolEvent, RegisteredProtocolSubstream};
+use bytes::BytesMut;
+use futures::prelude::*;
+use futures03::{compat::Compat, TryFutureExt as _};
+use futures_timer::Delay;
+use libp2p::core::{ConnectedPoint, PeerId, Endpoint};
+use libp2p::core::upgrade::{InboundUpgrade, OutboundUpgrade};
+use libp2p::swarm::{
+	ProtocolsHandler, ProtocolsHandlerEvent,
+	IntoProtocolsHandler,
+	KeepAlive,
+	ProtocolsHandlerUpgrErr,
+	SubstreamProtocol,
+};
+use log::{debug, error};
+use smallvec::{smallvec, SmallVec};
+use std::{borrow::Cow, error, fmt, io, marker::PhantomData, mem, time::Duration};
+use tokio_io::{AsyncRead, AsyncWrite};
+
+/// Implements the `IntoProtocolsHandler` trait of libp2p.
+///
+/// Every time a connection with a remote starts, an instance of this struct is created and
+/// sent to a background task dedicated to this connection. Once the connection is established,
+/// it is turned into a `CustomProtoHandler`. It then handles all communications that are specific
+/// to Substrate on that single connection.
+///
+/// Note that there can be multiple instance of this struct simultaneously for same peer. However
+/// if that happens, only one main instance can communicate with the outer layers of the code. In
+/// other words, the outer layers of the code only ever see one handler.
+///
+/// ## State of the handler
+///
+/// There are six possible states for the handler:
+///
+/// - Enabled and open, which is a normal operation.
+/// - Enabled and closed, in which case it will try to open substreams.
+/// - Disabled and open, in which case it will try to close substreams.
+/// - Disabled and closed, in which case the handler is idle. The connection will be
+///   garbage-collected after a few seconds if nothing more happens.
+/// - Initializing and open.
+/// - Initializing and closed, which is the state the handler starts in.
+///
+/// The Init/Enabled/Disabled state is entirely controlled by the user by sending `Enable` or
+/// `Disable` messages to the handler. The handler itself never transitions automatically between
+/// these states. For example, if the handler reports a network misbehaviour, it will close the
+/// substreams but it is the role of the user to send a `Disabled` event if it wants the connection
+/// to close. Otherwise, the handler will try to reopen substreams.
+/// The handler starts in the "Initializing" state and must be transitionned to Enabled or Disabled
+/// as soon as possible.
+///
+/// The Open/Closed state is decided by the handler and is reported with the `CustomProtocolOpen`
+/// and `CustomProtocolClosed` events. The `CustomMessage` event can only be generated if the
+/// handler is open.
+///
+/// ## How it works
+///
+/// When the handler is created, it is initially in the `Init` state and waits for either a
+/// `Disable` or an `Enable` message from the outer layer. At any time, the outer layer is free to
+/// toggle the handler between the disabled and enabled states.
+///
+/// When the handler switches to "enabled", it opens a substream and negotiates the protocol named
+/// `/substrate/xxx`, where `xxx` is chosen by the user and depends on the chain.
+///
+/// For backwards compatibility reasons, when we switch to "enabled" for the first time (while we
+/// are still in "init" mode) and we are the connection listener, we don't open a substream.
+///
+/// In order the handle the situation where both the remote and us get enabled at the same time,
+/// we tolerate multiple substreams open at the same time. Messages are transmitted on an arbitrary
+/// substream. The endpoints don't try to agree on a single substream.
+///
+/// We consider that we are now "closed" if the remote closes all the existing substreams.
+/// Re-opening it can then be performed by closing all active substream and re-opening one.
+///
+pub struct CustomProtoHandlerProto<TSubstream> {
+	/// Configuration for the protocol upgrade to negotiate.
+	protocol: RegisteredProtocol,
+
+	/// Marker to pin the generic type.
+	marker: PhantomData<TSubstream>,
+}
+
+impl<TSubstream> CustomProtoHandlerProto<TSubstream>
+where
+	TSubstream: AsyncRead + AsyncWrite,
+{
+	/// Builds a new `CustomProtoHandlerProto`.
+	pub fn new(protocol: RegisteredProtocol) -> Self {
+		CustomProtoHandlerProto {
+			protocol,
+			marker: PhantomData,
+		}
+	}
+}
+
+impl<TSubstream> IntoProtocolsHandler for CustomProtoHandlerProto<TSubstream>
+where
+	TSubstream: AsyncRead + AsyncWrite,
+{
+	type Handler = CustomProtoHandler<TSubstream>;
+
+	fn inbound_protocol(&self) -> RegisteredProtocol {
+		self.protocol.clone()
+	}
+
+	fn into_handler(self, remote_peer_id: &PeerId, connected_point: &ConnectedPoint) -> Self::Handler {
+		CustomProtoHandler {
+			protocol: self.protocol,
+			endpoint: connected_point.to_endpoint(),
+			remote_peer_id: remote_peer_id.clone(),
+			state: ProtocolState::Init {
+				substreams: SmallVec::new(),
+				init_deadline: Delay::new(Duration::from_secs(5)).compat()
+			},
+			events_queue: SmallVec::new(),
+		}
+	}
+}
+
+/// The actual handler once the connection has been established.
+pub struct CustomProtoHandler<TSubstream> {
+	/// Configuration for the protocol upgrade to negotiate.
+	protocol: RegisteredProtocol,
+
+	/// State of the communications with the remote.
+	state: ProtocolState<TSubstream>,
+
+	/// Identifier of the node we're talking to. Used only for logging purposes and shouldn't have
+	/// any influence on the behaviour.
+	remote_peer_id: PeerId,
+
+	/// Whether we are the connection dialer or listener. Used to determine who, between the local
+	/// node and the remote node, has priority.
+	endpoint: Endpoint,
+
+	/// Queue of events to send to the outside.
+	///
+	/// This queue must only ever be modified to insert elements at the back, or remove the first
+	/// element.
+	events_queue: SmallVec<[ProtocolsHandlerEvent<RegisteredProtocol, (), CustomProtoHandlerOut>; 16]>,
+}
+
+/// State of the handler.
+enum ProtocolState<TSubstream> {
+	/// Waiting for the behaviour to tell the handler whether it is enabled or disabled.
+	Init {
+		/// List of substreams opened by the remote but that haven't been processed yet.
+		substreams: SmallVec<[RegisteredProtocolSubstream<TSubstream>; 6]>,
+		/// Deadline after which the initialization is abnormally long.
+		init_deadline: Compat<Delay>,
+	},
+
+	/// Handler is opening a substream in order to activate itself.
+	/// If we are in this state, we haven't sent any `CustomProtocolOpen` yet.
+	Opening {
+		/// Deadline after which the opening is abnormally long.
+		deadline: Compat<Delay>,
+	},
+
+	/// Normal operating mode. Contains the substreams that are open.
+	/// If we are in this state, we have sent a `CustomProtocolOpen` message to the outside.
+	Normal {
+		/// The substreams where bidirectional communications happen.
+		substreams: SmallVec<[RegisteredProtocolSubstream<TSubstream>; 4]>,
+		/// Contains substreams which are being shut down.
+		shutdown: SmallVec<[RegisteredProtocolSubstream<TSubstream>; 4]>,
+	},
+
+	/// We are disabled. Contains substreams that are being closed.
+	/// If we are in this state, either we have sent a `CustomProtocolClosed` message to the
+	/// outside or we have never sent any `CustomProtocolOpen` in the first place.
+	Disabled {
+		/// List of substreams to shut down.
+		shutdown: SmallVec<[RegisteredProtocolSubstream<TSubstream>; 6]>,
+
+		/// If true, we should reactivate the handler after all the substreams in `shutdown` have
+		/// been closed.
+		///
+		/// Since we don't want to mix old and new substreams, we wait for all old substreams to
+		/// be closed before opening any new one.
+		reenable: bool,
+	},
+
+	/// In this state, we don't care about anything anymore and need to kill the connection as soon
+	/// as possible.
+	KillAsap,
+
+	/// We sometimes temporarily switch to this state during processing. If we are in this state
+	/// at the beginning of a method, that means something bad happened in the source code.
+	Poisoned,
+}
+
+/// Event that can be received by a `CustomProtoHandler`.
+#[derive(Debug)]
+pub enum CustomProtoHandlerIn {
+	/// The node should start using custom protocols.
+	Enable,
+
+	/// The node should stop using custom protocols.
+	Disable,
+
+	/// Sends a message through a custom protocol substream.
+	SendCustomMessage {
+		/// The message to send.
+		message: Vec<u8>,
+	},
+}
+
+/// Event that can be emitted by a `CustomProtoHandler`.
+#[derive(Debug)]
+pub enum CustomProtoHandlerOut {
+	/// Opened a custom protocol with the remote.
+	CustomProtocolOpen {
+		/// Version of the protocol that has been opened.
+		version: u8,
+	},
+
+	/// Closed a custom protocol with the remote.
+	CustomProtocolClosed {
+		/// Reason why the substream closed, for diagnostic purposes.
+		reason: Cow<'static, str>,
+	},
+
+	/// Receives a message on a custom protocol substream.
+	CustomMessage {
+		/// Message that has been received.
+		message: BytesMut,
+	},
+
+	/// A substream to the remote is clogged. The send buffer is very large, and we should print
+	/// a diagnostic message and/or avoid sending more data.
+	Clogged {
+		/// Copy of the messages that are within the buffer, for further diagnostic.
+		messages: Vec<Vec<u8>>,
+	},
+
+	/// An error has happened on the protocol level with this node.
+	ProtocolError {
+		/// If true the error is severe, such as a protocol violation.
+		is_severe: bool,
+		/// The error that happened.
+		error: Box<dyn error::Error + Send + Sync>,
+	},
+}
+
+impl<TSubstream> CustomProtoHandler<TSubstream>
+where
+	TSubstream: AsyncRead + AsyncWrite,
+{
+	/// Enables the handler.
+	fn enable(&mut self) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { substreams: incoming, .. } => {
+				if incoming.is_empty() {
+					if let Endpoint::Dialer = self.endpoint {
+						self.events_queue.push(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+							protocol: SubstreamProtocol::new(self.protocol.clone()),
+							info: (),
+						});
+					}
+					ProtocolState::Opening {
+						deadline: Delay::new(Duration::from_secs(60)).compat()
+					}
+
+				} else {
+					let event = CustomProtoHandlerOut::CustomProtocolOpen {
+						version: incoming[0].protocol_version()
+					};
+					self.events_queue.push(ProtocolsHandlerEvent::Custom(event));
+					ProtocolState::Normal {
+						substreams: incoming.into_iter().collect(),
+						shutdown: SmallVec::new()
+					}
+				}
+			}
+
+			st @ ProtocolState::KillAsap => st,
+			st @ ProtocolState::Opening { .. } => st,
+			st @ ProtocolState::Normal { .. } => st,
+			ProtocolState::Disabled { shutdown, .. } => {
+				ProtocolState::Disabled { shutdown, reenable: true }
+			}
+		}
+	}
+
+	/// Disables the handler.
+	fn disable(&mut self) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { substreams: mut shutdown, .. } => {
+				for s in &mut shutdown {
+					s.shutdown();
+				}
+				ProtocolState::Disabled { shutdown, reenable: false }
+			}
+
+			ProtocolState::Opening { .. } | ProtocolState::Normal { .. } =>
+				// At the moment, if we get disabled while things were working, we kill the entire
+				// connection in order to force a reset of the state.
+				// This is obviously an extremely shameful way to do things, but at the time of
+				// the writing of this comment, the networking works very poorly and a solution
+				// needs to be found.
+				ProtocolState::KillAsap,
+
+			ProtocolState::Disabled { shutdown, .. } =>
+				ProtocolState::Disabled { shutdown, reenable: false },
+
+			ProtocolState::KillAsap => ProtocolState::KillAsap,
+		};
+	}
+
+	/// Polls the state for events. Optionally returns an event to produce.
+	#[must_use]
+	fn poll_state(&mut self)
+		-> Option<ProtocolsHandlerEvent<RegisteredProtocol, (), CustomProtoHandlerOut>> {
+		match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				self.state = ProtocolState::Poisoned;
+				None
+			}
+
+			ProtocolState::Init { substreams, mut init_deadline } => {
+				match init_deadline.poll() {
+					Ok(Async::Ready(())) => {
+						init_deadline = Delay::new(Duration::from_secs(60)).compat();
+						error!(target: "sub-libp2p", "Handler initialization process is too long \
+							with {:?}", self.remote_peer_id)
+					},
+					Ok(Async::NotReady) => {}
+					Err(_) => error!(target: "sub-libp2p", "Tokio timer has errored")
+				}
+
+				self.state = ProtocolState::Init { substreams, init_deadline };
+				None
+			}
+
+			ProtocolState::Opening { mut deadline } => {
+				match deadline.poll() {
+					Ok(Async::Ready(())) => {
+						deadline = Delay::new(Duration::from_secs(60)).compat();
+						let event = CustomProtoHandlerOut::ProtocolError {
+							is_severe: true,
+							error: "Timeout when opening protocol".to_string().into(),
+						};
+						self.state = ProtocolState::Opening { deadline };
+						Some(ProtocolsHandlerEvent::Custom(event))
+					},
+					Ok(Async::NotReady) => {
+						self.state = ProtocolState::Opening { deadline };
+						None
+					},
+					Err(_) => {
+						error!(target: "sub-libp2p", "Tokio timer has errored");
+						deadline = Delay::new(Duration::from_secs(60)).compat();
+						self.state = ProtocolState::Opening { deadline };
+						None
+					},
+				}
+			}
+
+			ProtocolState::Normal { mut substreams, mut shutdown } => {
+				for n in (0..substreams.len()).rev() {
+					let mut substream = substreams.swap_remove(n);
+					match substream.poll() {
+						Ok(Async::NotReady) => substreams.push(substream),
+						Ok(Async::Ready(Some(RegisteredProtocolEvent::Message(message)))) => {
+							let event = CustomProtoHandlerOut::CustomMessage {
+								message
+							};
+							substreams.push(substream);
+							self.state = ProtocolState::Normal { substreams, shutdown };
+							return Some(ProtocolsHandlerEvent::Custom(event));
+						},
+						Ok(Async::Ready(Some(RegisteredProtocolEvent::Clogged { messages }))) => {
+							let event = CustomProtoHandlerOut::Clogged {
+								messages,
+							};
+							substreams.push(substream);
+							self.state = ProtocolState::Normal { substreams, shutdown };
+							return Some(ProtocolsHandlerEvent::Custom(event));
+						}
+						Ok(Async::Ready(None)) => {
+							shutdown.push(substream);
+							if substreams.is_empty() {
+								let event = CustomProtoHandlerOut::CustomProtocolClosed {
+									reason: "All substreams have been closed by the remote".into(),
+								};
+								self.state = ProtocolState::Disabled {
+									shutdown: shutdown.into_iter().collect(),
+									reenable: true
+								};
+								return Some(ProtocolsHandlerEvent::Custom(event));
+							}
+						}
+						Err(err) => {
+							if substreams.is_empty() {
+								let event = CustomProtoHandlerOut::CustomProtocolClosed {
+									reason: format!("Error on the last substream: {:?}", err).into(),
+								};
+								self.state = ProtocolState::Disabled {
+									shutdown: shutdown.into_iter().collect(),
+									reenable: true
+								};
+								return Some(ProtocolsHandlerEvent::Custom(event));
+							} else {
+								debug!(target: "sub-libp2p", "Error on extra substream: {:?}", err);
+							}
+						}
+					}
+				}
+
+				// This code is reached is none if and only if none of the substreams are in a ready state.
+				self.state = ProtocolState::Normal { substreams, shutdown };
+				None
+			}
+
+			ProtocolState::Disabled { mut shutdown, reenable } => {
+				shutdown_list(&mut shutdown);
+				// If `reenable` is `true`, that means we should open the substreams system again
+				// after all the substreams are closed.
+				if reenable && shutdown.is_empty() {
+					self.state = ProtocolState::Opening {
+						deadline: Delay::new(Duration::from_secs(60)).compat()
+					};
+					Some(ProtocolsHandlerEvent::OutboundSubstreamRequest {
+						protocol: SubstreamProtocol::new(self.protocol.clone()),
+						info: (),
+					})
+				} else {
+					self.state = ProtocolState::Disabled { shutdown, reenable };
+					None
+				}
+			}
+
+			ProtocolState::KillAsap => None,
+		}
+	}
+
+	/// Called by `inject_fully_negotiated_inbound` and `inject_fully_negotiated_outbound`.
+	fn inject_fully_negotiated(
+		&mut self,
+		mut substream: RegisteredProtocolSubstream<TSubstream>
+	) {
+		self.state = match mem::replace(&mut self.state, ProtocolState::Poisoned) {
+			ProtocolState::Poisoned => {
+				error!(target: "sub-libp2p", "Handler with {:?} is in poisoned state",
+					self.remote_peer_id);
+				ProtocolState::Poisoned
+			}
+
+			ProtocolState::Init { mut substreams, init_deadline } => {
+				if substream.endpoint() == Endpoint::Dialer {
+					error!(target: "sub-libp2p", "Opened dialing substream with {:?} before \
+						initialization", self.remote_peer_id);
+				}
+				substreams.push(substream);
+				ProtocolState::Init { substreams, init_deadline }
+			}
+
+			ProtocolState::Opening { .. } => {
+				let event = CustomProtoHandlerOut::CustomProtocolOpen {
+					version: substream.protocol_version()
+				};
+				self.events_queue.push(ProtocolsHandlerEvent::Custom(event));
+				ProtocolState::Normal {
+					substreams: smallvec![substream],
+					shutdown: SmallVec::new()
+				}
+			}
+
+			ProtocolState::Normal { substreams: mut existing, shutdown } => {
+				existing.push(substream);
+				ProtocolState::Normal { substreams: existing, shutdown }
+			}
+
+			ProtocolState::Disabled { mut shutdown, .. } => {
+				substream.shutdown();
+				shutdown.push(substream);
+				ProtocolState::Disabled { shutdown, reenable: false }
+			}
+
+			ProtocolState::KillAsap => ProtocolState::KillAsap,
+		};
+	}
+
+	/// Sends a message to the remote.
+	fn send_message(&mut self, message: Vec<u8>) {
+		match self.state {
+			ProtocolState::Normal { ref mut substreams, .. } =>
+				substreams[0].send_message(message),
+
+			_ => debug!(target: "sub-libp2p", "Tried to send message over closed protocol \
+				with {:?}", self.remote_peer_id)
+		}
+	}
+}
+
+impl<TSubstream> ProtocolsHandler for CustomProtoHandler<TSubstream>
+where TSubstream: AsyncRead + AsyncWrite {
+	type InEvent = CustomProtoHandlerIn;
+	type OutEvent = CustomProtoHandlerOut;
+	type Substream = TSubstream;
+	type Error = ConnectionKillError;
+	type InboundProtocol = RegisteredProtocol;
+	type OutboundProtocol = RegisteredProtocol;
+	type OutboundOpenInfo = ();
+
+	fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
+		SubstreamProtocol::new(self.protocol.clone())
+	}
+
+	fn inject_fully_negotiated_inbound(
+		&mut self,
+		proto: <Self::InboundProtocol as InboundUpgrade<TSubstream>>::Output
+	) {
+		self.inject_fully_negotiated(proto);
+	}
+
+	fn inject_fully_negotiated_outbound(
+		&mut self,
+		proto: <Self::OutboundProtocol as OutboundUpgrade<TSubstream>>::Output,
+		_: Self::OutboundOpenInfo
+	) {
+		self.inject_fully_negotiated(proto);
+	}
+
+	fn inject_event(&mut self, message: CustomProtoHandlerIn) {
+		match message {
+			CustomProtoHandlerIn::Disable => self.disable(),
+			CustomProtoHandlerIn::Enable => self.enable(),
+			CustomProtoHandlerIn::SendCustomMessage { message } =>
+				self.send_message(message),
+		}
+	}
+
+	#[inline]
+	fn inject_dial_upgrade_error(&mut self, _: (), err: ProtocolsHandlerUpgrErr<io::Error>) {
+		let is_severe = match err {
+			ProtocolsHandlerUpgrErr::Upgrade(_) => true,
+			_ => false,
+		};
+
+		self.events_queue.push(ProtocolsHandlerEvent::Custom(CustomProtoHandlerOut::ProtocolError {
+			is_severe,
+			error: Box::new(err),
+		}));
+	}
+
+	fn connection_keep_alive(&self) -> KeepAlive {
+		match self.state {
+			ProtocolState::Init { .. } | ProtocolState::Opening { .. } |
+			ProtocolState::Normal { .. } => KeepAlive::Yes,
+			ProtocolState::Disabled { .. } | ProtocolState::Poisoned |
+	  		ProtocolState::KillAsap => KeepAlive::No,
+		}
+	}
+
+	fn poll(
+		&mut self,
+	) -> Poll<
+		ProtocolsHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::OutEvent>,
+		Self::Error,
+	> {
+		// Flush the events queue if necessary.
+		if !self.events_queue.is_empty() {
+			let event = self.events_queue.remove(0);
+			return Ok(Async::Ready(event))
+		}
+
+		// Kill the connection if needed.
+		if let ProtocolState::KillAsap = self.state {
+			return Err(ConnectionKillError);
+		}
+
+		// Process all the substreams.
+		if let Some(event) = self.poll_state() {
+			return Ok(Async::Ready(event))
+		}
+
+		Ok(Async::NotReady)
+	}
+}
+
+impl<TSubstream> fmt::Debug for CustomProtoHandler<TSubstream>
+where
+	TSubstream: AsyncRead + AsyncWrite,
+{
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		f.debug_struct("CustomProtoHandler")
+			.finish()
+	}
+}
+
+/// Given a list of substreams, tries to shut them down. The substreams that have been successfully
+/// shut down are removed from the list.
+fn shutdown_list<TSubstream>
+	(list: &mut SmallVec<impl smallvec::Array<Item = RegisteredProtocolSubstream<TSubstream>>>)
+where TSubstream: AsyncRead + AsyncWrite {
+	'outer: for n in (0..list.len()).rev() {
+		let mut substream = list.swap_remove(n);
+		loop {
+			match substream.poll() {
+				Ok(Async::Ready(Some(_))) => {}
+				Ok(Async::NotReady) => break,
+				Err(_) | Ok(Async::Ready(None)) => continue 'outer,
+			}
+		}
+		list.push(substream);
+	}
+}
+
+/// Error returned when switching from normal to disabled.
+#[derive(Debug)]
+pub struct ConnectionKillError;
+
+impl error::Error for ConnectionKillError {
+}
+
+impl fmt::Display for ConnectionKillError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "Connection kill when switching from normal to disabled")
+	}
+}

substrate/client/network/src/legacy_proto/mod.rs

@@ -0,0 +1,22 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+pub use self::behaviour::{LegacyProto, LegacyProtoOut};
+
+mod behaviour;
+mod handler;
+mod upgrade;
+mod tests;

substrate/client/network/src/legacy_proto/tests.rs

@@ -0,0 +1,410 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![cfg(test)]
+
+use futures::{future, prelude::*, try_ready};
+use codec::{Encode, Decode};
+use libp2p::core::nodes::Substream;
+use libp2p::core::{ConnectedPoint, transport::boxed::Boxed, muxing::StreamMuxerBox};
+use libp2p::swarm::{Swarm, ProtocolsHandler, IntoProtocolsHandler};
+use libp2p::swarm::{PollParameters, NetworkBehaviour, NetworkBehaviourAction};
+use libp2p::{PeerId, Multiaddr, Transport};
+use rand::seq::SliceRandom;
+use std::{io, time::Duration, time::Instant};
+use crate::message::Message;
+use crate::legacy_proto::{LegacyProto, LegacyProtoOut};
+use test_client::runtime::Block;
+
+/// Builds two nodes that have each other as bootstrap nodes.
+/// This is to be used only for testing, and a panic will happen if something goes wrong.
+fn build_nodes()
+-> (
+	Swarm<Boxed<(PeerId, StreamMuxerBox), io::Error>, CustomProtoWithAddr>,
+	Swarm<Boxed<(PeerId, StreamMuxerBox), io::Error>, CustomProtoWithAddr>
+) {
+	let mut out = Vec::with_capacity(2);
+
+	let keypairs: Vec<_> = (0..2).map(|_| libp2p::identity::Keypair::generate_ed25519()).collect();
+	let addrs: Vec<Multiaddr> = (0..2)
+		.map(|_| format!("/memory/{}", rand::random::<u64>()).parse().unwrap())
+		.collect();
+
+	for index in 0 .. 2 {
+		let keypair = keypairs[index].clone();
+		let transport = libp2p::core::transport::MemoryTransport
+			.and_then(move |out, endpoint| {
+				let secio = libp2p::secio::SecioConfig::new(keypair);
+				libp2p::core::upgrade::apply(
+					out,
+					secio,
+					endpoint,
+					libp2p::core::upgrade::Version::V1
+				)
+			})
+			.and_then(move |(peer_id, stream), endpoint| {
+				libp2p::core::upgrade::apply(
+					stream,
+					libp2p::yamux::Config::default(),
+					endpoint,
+					libp2p::core::upgrade::Version::V1
+				)
+					.map(|muxer| (peer_id, libp2p::core::muxing::StreamMuxerBox::new(muxer)))
+			})
+			.timeout(Duration::from_secs(20))
+			.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+			.boxed();
+
+		let (peerset, _) = peerset::Peerset::from_config(peerset::PeersetConfig {
+			in_peers: 25,
+			out_peers: 25,
+			bootnodes: if index == 0 {
+				keypairs
+					.iter()
+					.skip(1)
+					.map(|keypair| keypair.public().into_peer_id())
+					.collect()
+			} else {
+				vec![]
+			},
+			reserved_only: false,
+			reserved_nodes: Vec::new(),
+		});
+
+		let behaviour = CustomProtoWithAddr {
+			inner: LegacyProto::new(&b"test"[..], &[1], peerset),
+			addrs: addrs
+				.iter()
+				.enumerate()
+				.filter_map(|(n, a)| if n != index {
+					Some((keypairs[n].public().into_peer_id(), a.clone()))
+				} else {
+					None
+				})
+				.collect(),
+		};
+
+		let mut swarm = Swarm::new(
+			transport,
+			behaviour,
+			keypairs[index].public().into_peer_id()
+		);
+		Swarm::listen_on(&mut swarm, addrs[index].clone()).unwrap();
+		out.push(swarm);
+	}
+
+	// Final output
+	let mut out_iter = out.into_iter();
+	let first = out_iter.next().unwrap();
+	let second = out_iter.next().unwrap();
+	(first, second)
+}
+
+/// Wraps around the `CustomBehaviour` network behaviour, and adds hardcoded node addresses to it.
+struct CustomProtoWithAddr {
+	inner: LegacyProto<Substream<StreamMuxerBox>>,
+	addrs: Vec<(PeerId, Multiaddr)>,
+}
+
+impl std::ops::Deref for CustomProtoWithAddr {
+	type Target = LegacyProto<Substream<StreamMuxerBox>>;
+
+	fn deref(&self) -> &Self::Target {
+		&self.inner
+	}
+}
+
+impl std::ops::DerefMut for CustomProtoWithAddr {
+	fn deref_mut(&mut self) -> &mut Self::Target {
+		&mut self.inner
+	}
+}
+
+impl NetworkBehaviour for CustomProtoWithAddr {
+	type ProtocolsHandler =
+		<LegacyProto<Substream<StreamMuxerBox>> as NetworkBehaviour>::ProtocolsHandler;
+	type OutEvent = <LegacyProto<Substream<StreamMuxerBox>> as NetworkBehaviour>::OutEvent;
+
+	fn new_handler(&mut self) -> Self::ProtocolsHandler {
+		self.inner.new_handler()
+	}
+
+	fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
+		let mut list = self.inner.addresses_of_peer(peer_id);
+		for (p, a) in self.addrs.iter() {
+			if p == peer_id {
+				list.push(a.clone());
+			}
+		}
+		list
+	}
+
+	fn inject_connected(&mut self, peer_id: PeerId, endpoint: ConnectedPoint) {
+		self.inner.inject_connected(peer_id, endpoint)
+	}
+
+	fn inject_disconnected(&mut self, peer_id: &PeerId, endpoint: ConnectedPoint) {
+		self.inner.inject_disconnected(peer_id, endpoint)
+	}
+
+	fn inject_node_event(
+		&mut self,
+		peer_id: PeerId,
+		event: <<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::OutEvent
+	) {
+		self.inner.inject_node_event(peer_id, event)
+	}
+
+	fn poll(
+		&mut self,
+		params: &mut impl PollParameters
+	) -> Async<
+		NetworkBehaviourAction<
+			<<Self::ProtocolsHandler as IntoProtocolsHandler>::Handler as ProtocolsHandler>::InEvent,
+			Self::OutEvent
+		>
+	> {
+		self.inner.poll(params)
+	}
+
+	fn inject_replaced(&mut self, peer_id: PeerId, closed_endpoint: ConnectedPoint, new_endpoint: ConnectedPoint) {
+		self.inner.inject_replaced(peer_id, closed_endpoint, new_endpoint)
+	}
+
+	fn inject_addr_reach_failure(&mut self, peer_id: Option<&PeerId>, addr: &Multiaddr, error: &dyn std::error::Error) {
+		self.inner.inject_addr_reach_failure(peer_id, addr, error)
+	}
+
+	fn inject_dial_failure(&mut self, peer_id: &PeerId) {
+		self.inner.inject_dial_failure(peer_id)
+	}
+
+	fn inject_new_listen_addr(&mut self, addr: &Multiaddr) {
+		self.inner.inject_new_listen_addr(addr)
+	}
+
+	fn inject_expired_listen_addr(&mut self, addr: &Multiaddr) {
+		self.inner.inject_expired_listen_addr(addr)
+	}
+
+	fn inject_new_external_addr(&mut self, addr: &Multiaddr) {
+		self.inner.inject_new_external_addr(addr)
+	}
+}
+
+#[test]
+fn two_nodes_transfer_lots_of_packets() {
+	// We spawn two nodes, then make the first one send lots of packets to the second one. The test
+	// ends when the second one has received all of them.
+
+	// Note that if we go too high, we will reach the limit to the number of simultaneous
+	// substreams allowed by the multiplexer.
+	const NUM_PACKETS: u32 = 5000;
+
+	let (mut service1, mut service2) = build_nodes();
+
+	let fut1 = future::poll_fn(move || -> io::Result<_> {
+		loop {
+			match try_ready!(service1.poll()) {
+				Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => {
+					for n in 0 .. NUM_PACKETS {
+						service1.send_packet(
+							&peer_id,
+							Message::<Block>::ChainSpecific(vec![(n % 256) as u8]).encode()
+						);
+					}
+				},
+				_ => panic!(),
+			}
+		}
+	});
+
+	let mut packet_counter = 0u32;
+	let fut2 = future::poll_fn(move || -> io::Result<_> {
+		loop {
+			match try_ready!(service2.poll()) {
+				Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {},
+				Some(LegacyProtoOut::CustomMessage { message, .. }) => {
+					match Message::<Block>::decode(&mut &message[..]).unwrap() {
+						Message::<Block>::ChainSpecific(message) => {
+							assert_eq!(message.len(), 1);
+							packet_counter += 1;
+							if packet_counter == NUM_PACKETS {
+								return Ok(Async::Ready(()))
+							}
+						},
+						_ => panic!(),
+					}
+				}
+				_ => panic!(),
+			}
+		}
+	});
+
+	let combined = fut1.select(fut2).map_err(|(err, _)| err);
+	let _ = tokio::runtime::Runtime::new().unwrap().block_on(combined).unwrap();
+}
+
+#[test]
+fn basic_two_nodes_requests_in_parallel() {
+	let (mut service1, mut service2) = build_nodes();
+
+	// Generate random messages with or without a request id.
+	let mut to_send = {
+		let mut to_send = Vec::new();
+		for _ in 0..200 { // Note: don't make that number too high or the CPU usage will explode.
+			let msg = (0..10).map(|_| rand::random::<u8>()).collect::<Vec<_>>();
+			to_send.push(Message::<Block>::ChainSpecific(msg));
+		}
+		to_send
+	};
+
+	// Clone `to_send` in `to_receive`. Below we will remove from `to_receive` the messages we
+	// receive, until the list is empty.
+	let mut to_receive = to_send.clone();
+	to_send.shuffle(&mut rand::thread_rng());
+
+	let fut1 = future::poll_fn(move || -> io::Result<_> {
+		loop {
+			match try_ready!(service1.poll()) {
+				Some(LegacyProtoOut::CustomProtocolOpen { peer_id, .. }) => {
+					for msg in to_send.drain(..) {
+						service1.send_packet(&peer_id, msg.encode());
+					}
+				},
+				_ => panic!(),
+			}
+		}
+	});
+
+	let fut2 = future::poll_fn(move || -> io::Result<_> {
+		loop {
+			match try_ready!(service2.poll()) {
+				Some(LegacyProtoOut::CustomProtocolOpen { .. }) => {},
+				Some(LegacyProtoOut::CustomMessage { message, .. }) => {
+					let pos = to_receive.iter().position(|m| m.encode() == message).unwrap();
+					to_receive.remove(pos);
+					if to_receive.is_empty() {
+						return Ok(Async::Ready(()))
+					}
+				}
+				_ => panic!(),
+			}
+		}
+	});
+
+	let combined = fut1.select(fut2).map_err(|(err, _)| err);
+	let _ = tokio::runtime::Runtime::new().unwrap().block_on_all(combined).unwrap();
+}
+
+#[test]
+fn reconnect_after_disconnect() {
+	// We connect two nodes together, then force a disconnect (through the API of the `Service`),
+	// check that the disconnect worked, and finally check whether they successfully reconnect.
+
+	let (mut service1, mut service2) = build_nodes();
+
+	// We use the `current_thread` runtime because it doesn't require us to have `'static` futures.
+	let mut runtime = tokio::runtime::current_thread::Runtime::new().unwrap();
+
+	// For this test, the services can be in the following states.
+	#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+	enum ServiceState { NotConnected, FirstConnec, Disconnected, ConnectedAgain }
+	let mut service1_state = ServiceState::NotConnected;
+	let mut service2_state = ServiceState::NotConnected;
+
+	// Run the events loops.
+	runtime.block_on(future::poll_fn(|| -> Result<_, io::Error> {
+		loop {
+			let mut service1_not_ready = false;
+
+			match service1.poll().unwrap() {
+				Async::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => {
+					match service1_state {
+						ServiceState::NotConnected => {
+							service1_state = ServiceState::FirstConnec;
+							if service2_state == ServiceState::FirstConnec {
+								service1.disconnect_peer(Swarm::local_peer_id(&service2));
+							}
+						},
+						ServiceState::Disconnected => service1_state = ServiceState::ConnectedAgain,
+						ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(),
+					}
+				},
+				Async::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => {
+					match service1_state {
+						ServiceState::FirstConnec => service1_state = ServiceState::Disconnected,
+						ServiceState::ConnectedAgain| ServiceState::NotConnected |
+						ServiceState::Disconnected => panic!(),
+					}
+				},
+				Async::NotReady => service1_not_ready = true,
+				_ => panic!()
+			}
+
+			match service2.poll().unwrap() {
+				Async::Ready(Some(LegacyProtoOut::CustomProtocolOpen { .. })) => {
+					match service2_state {
+						ServiceState::NotConnected => {
+							service2_state = ServiceState::FirstConnec;
+							if service1_state == ServiceState::FirstConnec {
+								service1.disconnect_peer(Swarm::local_peer_id(&service2));
+							}
+						},
+						ServiceState::Disconnected => service2_state = ServiceState::ConnectedAgain,
+						ServiceState::FirstConnec | ServiceState::ConnectedAgain => panic!(),
+					}
+				},
+				Async::Ready(Some(LegacyProtoOut::CustomProtocolClosed { .. })) => {
+					match service2_state {
+						ServiceState::FirstConnec => service2_state = ServiceState::Disconnected,
+						ServiceState::ConnectedAgain| ServiceState::NotConnected |
+						ServiceState::Disconnected => panic!(),
+					}
+				},
+				Async::NotReady if service1_not_ready => break,
+				Async::NotReady => {}
+				_ => panic!()
+			}
+		}
+
+		if service1_state == ServiceState::ConnectedAgain && service2_state == ServiceState::ConnectedAgain {
+			Ok(Async::Ready(()))
+		} else {
+			Ok(Async::NotReady)
+		}
+	})).unwrap();
+
+	// Do a second 3-seconds run to make sure we don't get disconnected immediately again.
+	let mut delay = tokio::timer::Delay::new(Instant::now() + Duration::from_secs(3));
+	runtime.block_on(future::poll_fn(|| -> Result<_, io::Error> {
+		match service1.poll().unwrap() {
+			Async::NotReady => {},
+			_ => panic!()
+		}
+
+		match service2.poll().unwrap() {
+			Async::NotReady => {},
+			_ => panic!()
+		}
+
+		if let Async::Ready(()) = delay.poll().unwrap() {
+			Ok(Async::Ready(()))
+		} else {
+			Ok(Async::NotReady)
+		}
+	})).unwrap();
+}

substrate/client/network/src/legacy_proto/upgrade.rs

@@ -0,0 +1,292 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::config::ProtocolId;
+use bytes::{Bytes, BytesMut};
+use libp2p::core::{Negotiated, Endpoint, UpgradeInfo, InboundUpgrade, OutboundUpgrade, upgrade::ProtocolName};
+use libp2p::tokio_codec::Framed;
+use std::{collections::VecDeque, io, vec::IntoIter as VecIntoIter};
+use futures::{prelude::*, future, stream};
+use tokio_io::{AsyncRead, AsyncWrite};
+use unsigned_varint::codec::UviBytes;
+
+/// Connection upgrade for a single protocol.
+///
+/// Note that "a single protocol" here refers to `par` for example. However
+/// each protocol can have multiple different versions for networking purposes.
+pub struct RegisteredProtocol {
+	/// Id of the protocol for API purposes.
+	id: ProtocolId,
+	/// Base name of the protocol as advertised on the network.
+	/// Ends with `/` so that we can append a version number behind.
+	base_name: Bytes,
+	/// List of protocol versions that we support.
+	/// Ordered in descending order so that the best comes first.
+	supported_versions: Vec<u8>,
+}
+
+impl RegisteredProtocol {
+	/// Creates a new `RegisteredProtocol`. The `custom_data` parameter will be
+	/// passed inside the `RegisteredProtocolOutput`.
+	pub fn new(protocol: impl Into<ProtocolId>, versions: &[u8])
+		-> Self {
+		let protocol = protocol.into();
+		let mut base_name = Bytes::from_static(b"/substrate/");
+		base_name.extend_from_slice(protocol.as_bytes());
+		base_name.extend_from_slice(b"/");
+
+		RegisteredProtocol {
+			base_name,
+			id: protocol,
+			supported_versions: {
+				let mut tmp = versions.to_vec();
+				tmp.sort_unstable_by(|a, b| b.cmp(&a));
+				tmp
+			},
+		}
+	}
+}
+
+impl Clone for RegisteredProtocol {
+	fn clone(&self) -> Self {
+		RegisteredProtocol {
+			id: self.id.clone(),
+			base_name: self.base_name.clone(),
+			supported_versions: self.supported_versions.clone(),
+		}
+	}
+}
+
+/// Output of a `RegisteredProtocol` upgrade.
+pub struct RegisteredProtocolSubstream<TSubstream> {
+	/// If true, we are in the process of closing the sink.
+	is_closing: bool,
+	/// Whether the local node opened this substream (dialer), or we received this substream from
+	/// the remote (listener).
+	endpoint: Endpoint,
+	/// Buffer of packets to send.
+	send_queue: VecDeque<Vec<u8>>,
+	/// If true, we should call `poll_complete` on the inner sink.
+	requires_poll_complete: bool,
+	/// The underlying substream.
+	inner: stream::Fuse<Framed<Negotiated<TSubstream>, UviBytes<Vec<u8>>>>,
+	/// Version of the protocol that was negotiated.
+	protocol_version: u8,
+	/// If true, we have sent a "remote is clogged" event recently and shouldn't send another one
+	/// unless the buffer empties then fills itself again.
+	clogged_fuse: bool,
+}
+
+impl<TSubstream> RegisteredProtocolSubstream<TSubstream> {
+	/// Returns the version of the protocol that was negotiated.
+	pub fn protocol_version(&self) -> u8 {
+		self.protocol_version
+	}
+
+	/// Returns whether the local node opened this substream (dialer), or we received this
+	/// substream from the remote (listener).
+	pub fn endpoint(&self) -> Endpoint {
+		self.endpoint
+	}
+
+	/// Starts a graceful shutdown process on this substream.
+	///
+	/// Note that "graceful" means that we sent a closing message. We don't wait for any
+	/// confirmation from the remote.
+	///
+	/// After calling this, the stream is guaranteed to finish soon-ish.
+	pub fn shutdown(&mut self) {
+		self.is_closing = true;
+		self.send_queue.clear();
+	}
+
+	/// Sends a message to the substream.
+	pub fn send_message(&mut self, data: Vec<u8>) {
+		if self.is_closing {
+			return
+		}
+
+		self.send_queue.push_back(data);
+	}
+}
+
+/// Event produced by the `RegisteredProtocolSubstream`.
+#[derive(Debug, Clone)]
+pub enum RegisteredProtocolEvent {
+	/// Received a message from the remote.
+	Message(BytesMut),
+
+	/// Diagnostic event indicating that the connection is clogged and we should avoid sending too
+	/// many messages to it.
+	Clogged {
+		/// Copy of the messages that are within the buffer, for further diagnostic.
+		messages: Vec<Vec<u8>>,
+	},
+}
+
+impl<TSubstream> Stream for RegisteredProtocolSubstream<TSubstream>
+where TSubstream: AsyncRead + AsyncWrite {
+	type Item = RegisteredProtocolEvent;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
+		// Flushing the local queue.
+		while let Some(packet) = self.send_queue.pop_front() {
+			match self.inner.start_send(packet)? {
+				AsyncSink::NotReady(packet) => {
+					self.send_queue.push_front(packet);
+					break
+				},
+				AsyncSink::Ready => self.requires_poll_complete = true,
+			}
+		}
+
+		// If we are closing, close as soon as the Sink is closed.
+		if self.is_closing {
+			return Ok(self.inner.close()?.map(|()| None))
+		}
+
+		// Indicating that the remote is clogged if that's the case.
+		if self.send_queue.len() >= 2048 {
+			if !self.clogged_fuse {
+				// Note: this fuse is important not just for preventing us from flooding the logs;
+				// 	if you remove the fuse, then we will always return early from this function and
+				//	thus never read any message from the network.
+				self.clogged_fuse = true;
+				return Ok(Async::Ready(Some(RegisteredProtocolEvent::Clogged {
+					messages: self.send_queue.iter()
+						.map(|m| m.clone())
+						.collect(),
+				})))
+			}
+		} else {
+			self.clogged_fuse = false;
+		}
+
+		// Flushing if necessary.
+		if self.requires_poll_complete {
+			if let Async::Ready(()) = self.inner.poll_complete()? {
+				self.requires_poll_complete = false;
+			}
+		}
+
+		// Receiving incoming packets.
+		// Note that `inner` is wrapped in a `Fuse`, therefore we can poll it forever.
+		match self.inner.poll()? {
+			Async::Ready(Some(data)) => {
+				Ok(Async::Ready(Some(RegisteredProtocolEvent::Message(data))))
+			}
+			Async::Ready(None) =>
+				if !self.requires_poll_complete && self.send_queue.is_empty() {
+					Ok(Async::Ready(None))
+				} else {
+					Ok(Async::NotReady)
+				}
+			Async::NotReady => Ok(Async::NotReady),
+		}
+	}
+}
+
+impl UpgradeInfo for RegisteredProtocol {
+	type Info = RegisteredProtocolName;
+	type InfoIter = VecIntoIter<Self::Info>;
+
+	#[inline]
+	fn protocol_info(&self) -> Self::InfoIter {
+		// Report each version as an individual protocol.
+		self.supported_versions.iter().map(|&version| {
+			let num = version.to_string();
+
+			let mut name = self.base_name.clone();
+			name.extend_from_slice(num.as_bytes());
+			RegisteredProtocolName {
+				name,
+				version,
+			}
+		}).collect::<Vec<_>>().into_iter()
+	}
+}
+
+/// Implementation of `ProtocolName` for a custom protocol.
+#[derive(Debug, Clone)]
+pub struct RegisteredProtocolName {
+	/// Protocol name, as advertised on the wire.
+	name: Bytes,
+	/// Version number. Stored in string form in `name`, but duplicated here for easier retrieval.
+	version: u8,
+}
+
+impl ProtocolName for RegisteredProtocolName {
+	fn protocol_name(&self) -> &[u8] {
+		&self.name
+	}
+}
+
+impl<TSubstream> InboundUpgrade<TSubstream> for RegisteredProtocol
+where TSubstream: AsyncRead + AsyncWrite,
+{
+	type Output = RegisteredProtocolSubstream<TSubstream>;
+	type Future = future::FutureResult<Self::Output, io::Error>;
+	type Error = io::Error;
+
+	fn upgrade_inbound(
+		self,
+		socket: Negotiated<TSubstream>,
+		info: Self::Info,
+	) -> Self::Future {
+		let framed = {
+			let mut codec = UviBytes::default();
+			codec.set_max_len(16 * 1024 * 1024);		// 16 MiB hard limit for packets.
+			Framed::new(socket, codec)
+		};
+
+		future::ok(RegisteredProtocolSubstream {
+			is_closing: false,
+			endpoint: Endpoint::Listener,
+			send_queue: VecDeque::new(),
+			requires_poll_complete: false,
+			inner: framed.fuse(),
+			protocol_version: info.version,
+			clogged_fuse: false,
+		})
+	}
+}
+
+impl<TSubstream> OutboundUpgrade<TSubstream> for RegisteredProtocol
+where TSubstream: AsyncRead + AsyncWrite,
+{
+	type Output = <Self as InboundUpgrade<TSubstream>>::Output;
+	type Future = <Self as InboundUpgrade<TSubstream>>::Future;
+	type Error = <Self as InboundUpgrade<TSubstream>>::Error;
+
+	fn upgrade_outbound(
+		self,
+		socket: Negotiated<TSubstream>,
+		info: Self::Info,
+	) -> Self::Future {
+		let framed = Framed::new(socket, UviBytes::default());
+
+		future::ok(RegisteredProtocolSubstream {
+			is_closing: false,
+			endpoint: Endpoint::Dialer,
+			send_queue: VecDeque::new(),
+			requires_poll_complete: false,
+			inner: framed.fuse(),
+			protocol_version: info.version,
+			clogged_fuse: false,
+		})
+	}
+}

substrate/client/network/src/lib.rs

@@ -0,0 +1,307 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![warn(unused_extern_crates)]
+#![warn(missing_docs)]
+
+//! Substrate-specific P2P networking.
+//!
+//! **Important**: This crate is unstable and the API and usage may change.
+//!
+//! # Node identities and addresses
+//!
+//! In a decentralized network, each node possesses a network private key and a network public key.
+//! In Substrate, the keys are based on the ed25519 curve.
+//!
+//! From a node's public key, we can derive its *identity*. In Substrate and libp2p, a node's
+//! identity is represented with the [`PeerId`] struct. All network communications between nodes on
+//! the network use encryption derived from both sides's keys, which means that **identities cannot
+//! be faked**.
+//!
+//! A node's identity uniquely identifies a machine on the network. If you start two or more
+//! clients using the same network key, large interferences will happen.
+//!
+//! # Substrate's network protocol
+//!
+//! Substrate's networking protocol is based upon libp2p. It is at the moment not possible and not
+//! planned to permit using something else than the libp2p network stack and the rust-libp2p
+//! library. However the libp2p framework is very flexible and the rust-libp2p library could be
+//! extended to support a wider range of protocols than what is offered by libp2p.
+//!
+//! ## Discovery mechanisms
+//!
+//! In order for our node to join a peer-to-peer network, it has to know a list of nodes that are
+//! part of said network. This includes nodes identities and their address (how to reach them).
+//! Building such a list is called the **discovery** mechanism. There are three mechanisms that
+//! Substrate uses:
+//!
+//! - Bootstrap nodes. These are hard-coded node identities and addresses passed alongside with
+//! the network configuration.
+//! - mDNS. We perform a UDP broadcast on the local network. Nodes that listen may respond with
+//! their identity. More info [here](https://github.com/libp2p/specs/blob/master/discovery/mdns.md).
+//! mDNS can be disabled in the network configuration.
+//! - Kademlia random walk. Once connected, we perform random Kademlia `FIND_NODE` requests in
+//! order for nodes to propagate to us their view of the network. More information about Kademlia
+//! can be found [on Wikipedia](https://en.wikipedia.org/wiki/Kademlia).
+//!
+//! ## Connection establishment
+//!
+//! When node Alice knows node Bob's identity and address, it can establish a connection with Bob.
+//! All connections must always use encryption and multiplexing. While some node addresses (eg.
+//! addresses using `/quic`) already imply which encryption and/or multiplexing to use, for others
+//! the **multistream-select** protocol is used in order to negotiate an encryption layer and/or a
+//! multiplexing layer.
+//!
+//! The connection establishment mechanism is called the **transport**.
+//!
+//! As of the writing of this documentation, the following base-layer protocols are supported by
+//! Substrate:
+//!
+//! - TCP/IP for addresses of the form `/ip4/1.2.3.4/tcp/5`. Once the TCP connection is open, an
+//! encryption and a multiplexing layer are negotiated on top.
+//! - WebSockets for addresses of the form `/ip4/1.2.3.4/tcp/5/ws`. A TCP/IP connection is open and
+//! the WebSockets protocol is negotiated on top. Communications then happen inside WebSockets data
+//! frames. Encryption and multiplexing are additionally negotiated again inside this channel.
+//! - DNS for addresses of the form `/dns4/example.com/tcp/5` or `/dns4/example.com/tcp/5/ws`. A
+//! node's address can contain a domain name.
+//!
+//! The following encryption protocols are supported:
+//!
+//! - [Secio](https://github.com/libp2p/specs/tree/master/secio). A TLS-1.2-like protocol but
+//! without certificates. Support for secio will likely be deprecated in the far future.
+//! - [Noise](https://noiseprotocol.org/). Support for noise is very experimental. The details are
+//! very blurry and may change at any moment.
+//!
+//! The following multiplexing protocols are supported:
+//!
+//! - [Mplex](https://github.com/libp2p/specs/tree/master/mplex). Support for mplex will likely
+//! be deprecated in the future.
+//! - [Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md).
+//!
+//! ## Substreams
+//!
+//! Once a connection has been established and uses multiplexing, substreams can be opened. When
+//! a substream is open, the **multistream-select** protocol is used to negotiate which protocol to
+//! use on that given substream. In practice, Substrate opens the following substreams:
+//!
+//! - We periodically open an ephemeral substream in order to ping the remote and check whether the
+//! connection is still alive. Failure for the remote to reply leads to a disconnection. This uses
+//! the libp2p ping protocol.
+//! - We periodically open an ephemeral substream in order to ask information from the remote. This
+//! is called [the `identify` protocol](https://github.com/libp2p/specs/tree/master/identify).
+//! - We periodically open ephemeral substreams for Kademlia random walk queries. Each Kademlia
+//! query is done in a new separate substream. This uses the
+//! [standard libp2p Kademlia protocol](https://github.com/libp2p/specs/pull/108).
+//! - We optionally keep a substream alive for all Substrate-based communications. The name of the
+//! protocol negotiated is based on the *protocol ID* passed as part of the network configuration.
+//! This protocol ID should be unique for each chain and prevents nodes from different chains from
+//! connecting to each other. More information below.
+//!
+//! ## The Substrate substream
+//!
+//! Substrate uses a component named the **peerset manager (PSM)**. Through the discovery
+//! mechanism, the PSM is aware of the nodes that are part of the network and decides which nodes
+//! we should perform Substrate-based communications with. For these nodes, we open a connection
+//! if necessary and open a unique substream for Substrate-based communications. If the PSM decides
+//! that we should disconnect a node, then that substream is closed.
+//!
+//! For more information about the PSM, see the *substrate-peerset* crate.
+//!
+//! Note that at the moment there is no mechanism in place to solve the issues that arise where the
+//! two sides of a connection open the unique substream simultaneously. In order to not run into
+//! issues, only the dialer of a connection is allowed to open the unique substream. When the
+//! substream is closed, the entire connection is closed as well. This is a bug, and should be
+//! fixed by improving the protocol.
+//!
+//! Within the unique Substrate substream, messages encoded using
+//! [*parity-scale-codec*](https://github.com/paritytech/parity-scale-codec) are exchanged.
+//! The detail of theses messages is not totally in place, but they can be found in the
+//! `message.rs` file.
+//!
+//! Once the substream is open, the first step is an exchange of a *status* message from both
+//! sides, containing information such as the chain root hash, head of chain, and so on.
+//!
+//! Communications within this substream include:
+//!
+//! - Syncing. Blocks are announced and requested from other nodes.
+//! - Light-client requests. When a light client requires information, a random node we have a
+//! substream open with is chosen, and the information is requested from it.
+//! - Gossiping. Used for example by grandpa.
+//! - Network specialization. The network protocol can be specialized through a template parameter
+//! of the network service. This specialization is free to send and receive messages with the
+//! remote. This is meant to be used by the chain that is being built on top of Substrate
+//! (eg. Polkadot).
+//!
+//! It is intended that in the future each of these components gets more isolated, so that they
+//! are free to open and close their own substreams, and so that syncing and light client requests
+//! are able to communicate with nodes outside of the range of the PSM.
+//!
+//! # Usage
+//!
+//! Using the `substrate-network` crate is done through the [`NetworkWorker`] struct. Create this
+//! struct by passing a [`config::Params`], then poll it as if it was a `Future`. You can extract an
+//! `Arc<NetworkService>` from the `NetworkWorker`, which can be shared amongst multiple places
+//! in order to give orders to the networking.
+//!
+//! See the [`config`] module for more information about how to configure the networking.
+//!
+//! After the `NetworkWorker` has been created, the important things to do are:
+//!
+//! - Calling `NetworkWorker::poll` in order to advance the network.
+//! - Calling `on_block_import` whenever a block is added to the client.
+//! - Calling `on_block_finalized` whenever a block is finalized.
+//! - Calling `trigger_repropagate` when a transaction is added to the pool.
+//!
+//! More precise usage details are still being worked on and will likely change in the future.
+//!
+
+mod behaviour;
+mod chain;
+mod legacy_proto;
+mod debug_info;
+mod discovery;
+mod on_demand_layer;
+mod protocol;
+mod service;
+mod transport;
+
+pub mod config;
+pub mod error;
+
+#[cfg(any(test, feature = "test-helpers"))]
+pub mod test;
+
+pub use chain::{Client as ClientHandle, FinalityProofProvider};
+pub use service::{
+	NetworkService, NetworkWorker, TransactionPool, ExHashT, ReportHandle,
+	NetworkStateInfo,
+};
+pub use protocol::{PeerInfo, Context, consensus_gossip, message, specialization};
+pub use protocol::event::{Event, DhtEvent};
+pub use protocol::sync::SyncState;
+pub use libp2p::{Multiaddr, PeerId};
+#[doc(inline)]
+pub use libp2p::multiaddr;
+
+pub use message::{generic as generic_message, RequestId, Status as StatusMessage};
+pub use on_demand_layer::{OnDemand, RemoteResponse};
+
+// Used by the `construct_simple_protocol!` macro.
+#[doc(hidden)]
+pub use sr_primitives::traits::Block as BlockT;
+
+use libp2p::core::ConnectedPoint;
+use serde::{Deserialize, Serialize};
+use slog_derive::SerdeValue;
+use std::{collections::{HashMap, HashSet}, time::Duration};
+
+/// Extension trait for `NetworkBehaviour` that also accepts discovering nodes.
+pub trait DiscoveryNetBehaviour {
+	/// Notify the protocol that we have learned about the existence of nodes.
+	///
+	/// Can (or most likely will) be called multiple times with the same `PeerId`s.
+	///
+	/// Also note that there is no notification for expired nodes. The implementer must add a TTL
+	/// system, or remove nodes that will fail to reach.
+	fn add_discovered_nodes(&mut self, nodes: impl Iterator<Item = PeerId>);
+}
+
+/// Returns general information about the networking.
+///
+/// Meant for general diagnostic purposes.
+///
+/// **Warning**: This API is not stable.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, SerdeValue)]
+#[serde(rename_all = "camelCase")]
+pub struct NetworkState {
+	/// PeerId of the local node.
+	pub peer_id: String,
+	/// List of addresses the node is currently listening on.
+	pub listened_addresses: HashSet<Multiaddr>,
+	/// List of addresses the node knows it can be reached as.
+	pub external_addresses: HashSet<Multiaddr>,
+	/// List of node we're connected to.
+	pub connected_peers: HashMap<String, NetworkStatePeer>,
+	/// List of node that we know of but that we're not connected to.
+	pub not_connected_peers: HashMap<String, NetworkStateNotConnectedPeer>,
+	/// Downloaded bytes per second averaged over the past few seconds.
+	pub average_download_per_sec: u64,
+	/// Uploaded bytes per second averaged over the past few seconds.
+	pub average_upload_per_sec: u64,
+	/// State of the peerset manager.
+	pub peerset: serde_json::Value,
+}
+
+/// Part of the `NetworkState` struct. Unstable.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct NetworkStatePeer {
+	/// How we are connected to the node.
+	pub endpoint: NetworkStatePeerEndpoint,
+	/// Node information, as provided by the node itself. Can be empty if not known yet.
+	pub version_string: Option<String>,
+	/// Latest ping duration with this node.
+	pub latest_ping_time: Option<Duration>,
+	/// If true, the peer is "enabled", which means that we try to open Substrate-related protocols
+	/// with this peer. If false, we stick to Kademlia and/or other network-only protocols.
+	pub enabled: bool,
+	/// If true, the peer is "open", which means that we have a Substrate-related protocol
+	/// with this peer.
+	pub open: bool,
+	/// List of addresses known for this node.
+	pub known_addresses: HashSet<Multiaddr>,
+}
+
+/// Part of the `NetworkState` struct. Unstable.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct NetworkStateNotConnectedPeer {
+	/// List of addresses known for this node.
+	pub known_addresses: HashSet<Multiaddr>,
+	/// Node information, as provided by the node itself, if we were ever connected to this node.
+	pub version_string: Option<String>,
+	/// Latest ping duration with this node, if we were ever connected to this node.
+	pub latest_ping_time: Option<Duration>,
+}
+
+/// Part of the `NetworkState` struct. Unstable.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum NetworkStatePeerEndpoint {
+	/// We are dialing the given address.
+	Dialing(Multiaddr),
+	/// We are listening.
+	Listening {
+		/// Local address of the connection.
+		local_addr: Multiaddr,
+		/// Address data is sent back to.
+		send_back_addr: Multiaddr,
+	},
+}
+
+impl From<ConnectedPoint> for NetworkStatePeerEndpoint {
+	fn from(endpoint: ConnectedPoint) -> Self {
+		match endpoint {
+			ConnectedPoint::Dialer { address } =>
+				NetworkStatePeerEndpoint::Dialing(address),
+			ConnectedPoint::Listener { local_addr, send_back_addr } =>
+				NetworkStatePeerEndpoint::Listening {
+					local_addr,
+					send_back_addr
+				}
+		}
+	}
+}

substrate/client/network/src/on_demand_layer.rs

@@ -0,0 +1,151 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! On-demand requests service.
+
+use crate::protocol::light_dispatch::RequestData;
+use std::collections::HashMap;
+use std::sync::Arc;
+use futures::{prelude::*, sync::mpsc, sync::oneshot};
+use futures03::compat::{Compat01As03, Future01CompatExt as _};
+use parking_lot::Mutex;
+use client_api::error::Error as ClientError;
+use client_api::{Fetcher, FetchChecker, RemoteHeaderRequest,
+	RemoteCallRequest, RemoteReadRequest, RemoteChangesRequest,
+	RemoteReadChildRequest, RemoteBodyRequest};
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+
+/// Implements the `Fetcher` trait of the client. Makes it possible for the light client to perform
+/// network requests for some state.
+///
+/// This implementation stores all the requests in a queue. The network, in parallel, is then
+/// responsible for pulling elements out of that queue and fulfilling them.
+pub struct OnDemand<B: BlockT> {
+	/// Objects that checks whether what has been retrieved is correct.
+	checker: Arc<dyn FetchChecker<B>>,
+
+	/// Queue of requests. Set to `Some` at initialization, then extracted by the network.
+	///
+	/// Note that a better alternative would be to use a MPMC queue here, and add a `poll` method
+	/// from the `OnDemand`. However there exists no popular implementation of MPMC channels in
+	/// asynchronous Rust at the moment
+	requests_queue: Mutex<Option<mpsc::UnboundedReceiver<RequestData<B>>>>,
+
+	/// Sending side of `requests_queue`.
+	requests_send: mpsc::UnboundedSender<RequestData<B>>,
+}
+
+impl<B: BlockT> OnDemand<B> where
+	B::Header: HeaderT,
+{
+	/// Creates new on-demand service.
+	pub fn new(checker: Arc<dyn FetchChecker<B>>) -> Self {
+		let (requests_send, requests_queue) = mpsc::unbounded();
+		let requests_queue = Mutex::new(Some(requests_queue));
+
+		OnDemand {
+			checker,
+			requests_queue,
+			requests_send,
+		}
+	}
+
+	/// Get checker reference.
+	pub fn checker(&self) -> &Arc<dyn FetchChecker<B>> {
+		&self.checker
+	}
+
+	/// Extracts the queue of requests.
+	///
+	/// Whenever one of the methods of the `Fetcher` trait is called, an element is pushed on this
+	/// channel.
+	///
+	/// If this function returns `None`, that means that the receiver has already been extracted in
+	/// the past, and therefore that something already handles the requests.
+	pub(crate) fn extract_receiver(&self) -> Option<mpsc::UnboundedReceiver<RequestData<B>>> {
+		self.requests_queue.lock().take()
+	}
+}
+
+impl<B> Fetcher<B> for OnDemand<B> where
+	B: BlockT,
+	B::Header: HeaderT,
+{
+	type RemoteHeaderResult = Compat01As03<RemoteResponse<B::Header>>;
+	type RemoteReadResult = Compat01As03<RemoteResponse<HashMap<Vec<u8>, Option<Vec<u8>>>>>;
+	type RemoteCallResult = Compat01As03<RemoteResponse<Vec<u8>>>;
+	type RemoteChangesResult = Compat01As03<RemoteResponse<Vec<(NumberFor<B>, u32)>>>;
+	type RemoteBodyResult = Compat01As03<RemoteResponse<Vec<B::Extrinsic>>>;
+
+	fn remote_header(&self, request: RemoteHeaderRequest<B::Header>) -> Self::RemoteHeaderResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteHeader(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+
+	fn remote_read(&self, request: RemoteReadRequest<B::Header>) -> Self::RemoteReadResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteRead(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+
+	fn remote_read_child(
+		&self,
+		request: RemoteReadChildRequest<B::Header>
+	) -> Self::RemoteReadResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteReadChild(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+
+	fn remote_call(&self, request: RemoteCallRequest<B::Header>) -> Self::RemoteCallResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteCall(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+
+	fn remote_changes(&self, request: RemoteChangesRequest<B::Header>) -> Self::RemoteChangesResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteChanges(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+
+	fn remote_body(&self, request: RemoteBodyRequest<B::Header>) -> Self::RemoteBodyResult {
+		let (sender, receiver) = oneshot::channel();
+		let _ = self.requests_send.unbounded_send(RequestData::RemoteBody(request, sender));
+		RemoteResponse { receiver }.compat()
+	}
+}
+
+/// Future for an on-demand remote call response.
+pub struct RemoteResponse<T> {
+	receiver: oneshot::Receiver<Result<T, ClientError>>,
+}
+
+impl<T> Future for RemoteResponse<T> {
+	type Item = T;
+	type Error = ClientError;
+
+	fn poll(&mut self) -> Poll<Self::Item, Self::Error> {
+		self.receiver.poll()
+			.map_err(|_| ClientError::RemoteFetchCancelled.into())
+			.and_then(|r| match r {
+				Async::Ready(Ok(ready)) => Ok(Async::Ready(ready)),
+				Async::Ready(Err(error)) => Err(error),
+				Async::NotReady => Ok(Async::NotReady),
+			})
+	}
+}

substrate/client/network/src/protocol/consensus_gossip.rs

@@ -0,0 +1,920 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utility for gossip of network messages between nodes.
+//! Handles chain-specific and standard BFT messages.
+//!
+//! Gossip messages are separated by two categories: "topics" and consensus engine ID.
+//! The consensus engine ID is sent over the wire with the message, while the topic is not,
+//! with the expectation that the topic can be derived implicitly from the content of the
+//! message, assuming it is valid.
+//!
+//! Topics are a single 32-byte tag associated with a message, used to group those messages
+//! in an opaque way. Consensus code can invoke `broadcast_topic` to attempt to send all messages
+//! under a single topic to all peers who don't have them yet, and `send_topic` to
+//! send all messages under a single topic to a specific peer.
+//!
+//! Each consensus engine ID must have an associated,
+//! registered `Validator` for all gossip messages. The primary role of this `Validator` is
+//! to process incoming messages from peers, and decide whether to discard them or process
+//! them. It also decides whether to re-broadcast the message.
+//!
+//! The secondary role of the `Validator` is to check if a message is allowed to be sent to a given
+//! peer. All messages, before being sent, will be checked against this filter.
+//! This enables the validator to use information it's aware of about connected peers to decide
+//! whether to send messages to them at any given moment in time - In particular, to wait until
+//! peers can accept and process the message before sending it.
+//!
+//! Lastly, the fact that gossip validators can decide not to rebroadcast messages
+//! opens the door for neighbor status packets to be baked into the gossip protocol.
+//! These status packets will typically contain light pieces of information
+//! used to inform peers of a current view of protocol state.
+
+use std::collections::{HashMap, HashSet, hash_map::Entry};
+use std::sync::Arc;
+use std::iter;
+use std::time;
+use log::{trace, debug};
+use futures03::channel::mpsc;
+use lru_cache::LruCache;
+use libp2p::PeerId;
+use sr_primitives::traits::{Block as BlockT, Hash, HashFor};
+use sr_primitives::ConsensusEngineId;
+pub use crate::message::generic::{Message, ConsensusMessage};
+use crate::protocol::Context;
+use crate::config::Roles;
+
+// FIXME: Add additional spam/DoS attack protection: https://github.com/paritytech/substrate/issues/1115
+const KNOWN_MESSAGES_CACHE_SIZE: usize = 4096;
+
+const REBROADCAST_INTERVAL: time::Duration = time::Duration::from_secs(30);
+/// Reputation change when a peer sends us a gossip message that we didn't know about.
+const GOSSIP_SUCCESS_REPUTATION_CHANGE: i32 = 1 << 4;
+/// Reputation change when a peer sends us a gossip message that we already knew about.
+const DUPLICATE_GOSSIP_REPUTATION_CHANGE: i32 = -(1 << 2);
+/// Reputation change when a peer sends us a gossip message for an unknown engine, whatever that
+/// means.
+const UNKNOWN_GOSSIP_REPUTATION_CHANGE: i32 = -(1 << 6);
+/// Reputation change when a peer sends a message from a topic it isn't registered on.
+const UNREGISTERED_TOPIC_REPUTATION_CHANGE: i32 = -(1 << 10);
+
+struct PeerConsensus<H> {
+	known_messages: HashSet<H>,
+	filtered_messages: HashMap<H, usize>,
+	roles: Roles,
+}
+
+/// Topic stream message with sender.
+#[derive(Debug, Eq, PartialEq)]
+pub struct TopicNotification {
+	/// Message data.
+	pub message: Vec<u8>,
+	/// Sender if available.
+	pub sender: Option<PeerId>,
+}
+
+struct MessageEntry<B: BlockT> {
+	message_hash: B::Hash,
+	topic: B::Hash,
+	message: ConsensusMessage,
+	sender: Option<PeerId>,
+}
+
+/// Consensus message destination.
+pub enum MessageRecipient {
+	/// Send to all peers.
+	BroadcastToAll,
+	/// Send to peers that don't have that message already.
+	BroadcastNew,
+	/// Send to specific peer.
+	Peer(PeerId),
+}
+
+/// The reason for sending out the message.
+#[derive(Eq, PartialEq, Copy, Clone)]
+#[cfg_attr(test, derive(Debug))]
+pub enum MessageIntent {
+	/// Requested broadcast.
+	Broadcast {
+		/// How many times this message was previously filtered by the gossip
+		/// validator when trying to propagate to a given peer.
+		previous_attempts: usize
+	},
+	/// Requested broadcast to all peers.
+	ForcedBroadcast,
+	/// Periodic rebroadcast of all messages to all peers.
+	PeriodicRebroadcast,
+}
+
+/// Message validation result.
+pub enum ValidationResult<H> {
+	/// Message should be stored and propagated under given topic.
+	ProcessAndKeep(H),
+	/// Message should be processed, but not propagated.
+	ProcessAndDiscard(H),
+	/// Message should be ignored.
+	Discard,
+}
+
+impl MessageIntent {
+	fn broadcast() -> MessageIntent {
+		MessageIntent::Broadcast { previous_attempts: 0 }
+	}
+}
+
+/// Validation context. Allows reacting to incoming messages by sending out further messages.
+pub trait ValidatorContext<B: BlockT> {
+	/// Broadcast all messages with given topic to peers that do not have it yet.
+	fn broadcast_topic(&mut self, topic: B::Hash, force: bool);
+	/// Broadcast a message to all peers that have not received it previously.
+	fn broadcast_message(&mut self, topic: B::Hash, message: Vec<u8>, force: bool);
+	/// Send addressed message to a peer.
+	fn send_message(&mut self, who: &PeerId, message: Vec<u8>);
+	/// Send all messages with given topic to a peer.
+	fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool);
+}
+
+struct NetworkContext<'g, 'p, B: BlockT> {
+	gossip: &'g mut ConsensusGossip<B>,
+	protocol: &'p mut dyn Context<B>,
+	engine_id: ConsensusEngineId,
+}
+
+impl<'g, 'p, B: BlockT> ValidatorContext<B> for NetworkContext<'g, 'p, B> {
+	/// Broadcast all messages with given topic to peers that do not have it yet.
+	fn broadcast_topic(&mut self, topic: B::Hash, force: bool) {
+		self.gossip.broadcast_topic(self.protocol, topic, force);
+	}
+
+	/// Broadcast a message to all peers that have not received it previously.
+	fn broadcast_message(&mut self, topic: B::Hash, message: Vec<u8>, force: bool) {
+		self.gossip.multicast(
+			self.protocol,
+			topic,
+			ConsensusMessage{ data: message, engine_id: self.engine_id.clone() },
+			force,
+		);
+	}
+
+	/// Send addressed message to a peer.
+	fn send_message(&mut self, who: &PeerId, message: Vec<u8>) {
+		self.protocol.send_consensus(who.clone(), vec![ConsensusMessage {
+			engine_id: self.engine_id,
+			data: message,
+		}]);
+	}
+
+	/// Send all messages with given topic to a peer.
+	fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool) {
+		self.gossip.send_topic(self.protocol, who, topic, self.engine_id, force);
+	}
+}
+
+fn propagate<'a, B: BlockT, I>(
+	protocol: &mut dyn Context<B>,
+	messages: I,
+	intent: MessageIntent,
+	peers: &mut HashMap<PeerId, PeerConsensus<B::Hash>>,
+	validators: &HashMap<ConsensusEngineId, Arc<dyn Validator<B>>>,
+)
+	where I: Clone + IntoIterator<Item=(&'a B::Hash, &'a B::Hash, &'a ConsensusMessage)>,  // (msg_hash, topic, message)
+{
+	let mut check_fns = HashMap::new();
+	let mut message_allowed = move |who: &PeerId, intent: MessageIntent, topic: &B::Hash, message: &ConsensusMessage| {
+		let engine_id = message.engine_id;
+		let check_fn = match check_fns.entry(engine_id) {
+			Entry::Occupied(entry) => entry.into_mut(),
+			Entry::Vacant(vacant) => match validators.get(&engine_id) {
+				None => return false, // treat all messages with no validator as not allowed
+				Some(validator) => vacant.insert(validator.message_allowed()),
+			}
+		};
+
+		(check_fn)(who, intent, topic, &message.data)
+	};
+
+	for (id, ref mut peer) in peers.iter_mut() {
+		let mut batch = Vec::new();
+		for (message_hash, topic, message) in messages.clone() {
+			let previous_attempts = peer.filtered_messages
+				.get(&message_hash)
+				.cloned()
+				.unwrap_or(0);
+
+			let intent = match intent {
+				MessageIntent::Broadcast { .. } =>
+					if peer.known_messages.contains(&message_hash) {
+						continue;
+					} else {
+						MessageIntent::Broadcast { previous_attempts }
+					},
+				MessageIntent::PeriodicRebroadcast =>
+					if peer.known_messages.contains(&message_hash) {
+						MessageIntent::PeriodicRebroadcast
+					} else {
+						// peer doesn't know message, so the logic should treat it as an
+						// initial broadcast.
+						MessageIntent::Broadcast { previous_attempts }
+					},
+				other => other,
+			};
+
+			if !message_allowed(id, intent, &topic, &message) {
+				let count = peer.filtered_messages
+					.entry(message_hash.clone())
+					.or_insert(0);
+
+				*count += 1;
+
+				continue;
+			}
+
+			peer.filtered_messages.remove(message_hash);
+			peer.known_messages.insert(message_hash.clone());
+
+			trace!(target: "gossip", "Propagating to {}: {:?}", id, message);
+			batch.push(message.clone())
+		}
+		protocol.send_consensus(id.clone(), batch);
+	}
+}
+
+/// Validates consensus messages.
+pub trait Validator<B: BlockT>: Send + Sync {
+	/// New peer is connected.
+	fn new_peer(&self, _context: &mut dyn ValidatorContext<B>, _who: &PeerId, _roles: Roles) {
+	}
+
+	/// New connection is dropped.
+	fn peer_disconnected(&self, _context: &mut dyn ValidatorContext<B>, _who: &PeerId) {
+	}
+
+	/// Validate consensus message.
+	fn validate(
+		&self,
+		context: &mut dyn ValidatorContext<B>,
+		sender: &PeerId,
+		data: &[u8]
+	) -> ValidationResult<B::Hash>;
+
+	/// Produce a closure for validating messages on a given topic.
+	fn message_expired<'a>(&'a self) -> Box<dyn FnMut(B::Hash, &[u8]) -> bool + 'a> {
+		Box::new(move |_topic, _data| false)
+	}
+
+	/// Produce a closure for filtering egress messages.
+	fn message_allowed<'a>(&'a self) -> Box<dyn FnMut(&PeerId, MessageIntent, &B::Hash, &[u8]) -> bool + 'a> {
+		Box::new(move |_who, _intent, _topic, _data| true)
+	}
+}
+
+/// Consensus network protocol handler. Manages statements and candidate requests.
+pub struct ConsensusGossip<B: BlockT> {
+	peers: HashMap<PeerId, PeerConsensus<B::Hash>>,
+	live_message_sinks: HashMap<(ConsensusEngineId, B::Hash), Vec<mpsc::UnboundedSender<TopicNotification>>>,
+	messages: Vec<MessageEntry<B>>,
+	known_messages: LruCache<B::Hash, ()>,
+	validators: HashMap<ConsensusEngineId, Arc<dyn Validator<B>>>,
+	next_broadcast: time::Instant,
+}
+
+impl<B: BlockT> ConsensusGossip<B> {
+	/// Create a new instance.
+	pub fn new() -> Self {
+		ConsensusGossip {
+			peers: HashMap::new(),
+			live_message_sinks: HashMap::new(),
+			messages: Default::default(),
+			known_messages: LruCache::new(KNOWN_MESSAGES_CACHE_SIZE),
+			validators: Default::default(),
+			next_broadcast: time::Instant::now() + REBROADCAST_INTERVAL,
+		}
+	}
+
+	/// Closes all notification streams.
+	pub fn abort(&mut self) {
+		self.live_message_sinks.clear();
+	}
+
+	/// Register message validator for a message type.
+	pub fn register_validator(
+		&mut self,
+		protocol: &mut dyn Context<B>,
+		engine_id: ConsensusEngineId,
+		validator: Arc<dyn Validator<B>>
+	) {
+		self.register_validator_internal(engine_id, validator.clone());
+		let peers: Vec<_> = self.peers.iter().map(|(id, peer)| (id.clone(), peer.roles)).collect();
+		for (id, roles) in peers {
+			let mut context = NetworkContext { gossip: self, protocol, engine_id: engine_id.clone() };
+			validator.new_peer(&mut context, &id, roles);
+		}
+	}
+
+	fn register_validator_internal(&mut self, engine_id: ConsensusEngineId, validator: Arc<dyn Validator<B>>) {
+		self.validators.insert(engine_id, validator.clone());
+	}
+
+	/// Handle new connected peer.
+	pub fn new_peer(&mut self, protocol: &mut dyn Context<B>, who: PeerId, roles: Roles) {
+		// light nodes are not valid targets for consensus gossip messages
+		if !roles.is_full() {
+			return;
+		}
+
+		trace!(target:"gossip", "Registering {:?} {}", roles, who);
+		self.peers.insert(who.clone(), PeerConsensus {
+			known_messages: HashSet::new(),
+			filtered_messages: HashMap::new(),
+			roles,
+		});
+		for (engine_id, v) in self.validators.clone() {
+			let mut context = NetworkContext { gossip: self, protocol, engine_id: engine_id.clone() };
+			v.new_peer(&mut context, &who, roles);
+		}
+	}
+
+	fn register_message_hashed(
+		&mut self,
+		message_hash: B::Hash,
+		topic: B::Hash,
+		message: ConsensusMessage,
+		sender: Option<PeerId>,
+	) {
+		if self.known_messages.insert(message_hash.clone(), ()).is_none() {
+			self.messages.push(MessageEntry {
+				message_hash,
+				topic,
+				message,
+				sender,
+			});
+		}
+	}
+
+	/// Registers a message without propagating it to any peers. The message
+	/// becomes available to new peers or when the service is asked to gossip
+	/// the message's topic. No validation is performed on the message, if the
+	/// message is already expired it should be dropped on the next garbage
+	/// collection.
+	pub fn register_message(
+		&mut self,
+		topic: B::Hash,
+		message: ConsensusMessage,
+	) {
+		let message_hash = HashFor::<B>::hash(&message.data[..]);
+		self.register_message_hashed(message_hash, topic, message, None);
+	}
+
+	/// Call when a peer has been disconnected to stop tracking gossip status.
+	pub fn peer_disconnected(&mut self, protocol: &mut dyn Context<B>, who: PeerId) {
+		for (engine_id, v) in self.validators.clone() {
+			let mut context = NetworkContext { gossip: self, protocol, engine_id: engine_id.clone() };
+			v.peer_disconnected(&mut context, &who);
+		}
+	}
+
+	/// Perform periodic maintenance
+	pub fn tick(&mut self, protocol: &mut dyn Context<B>) {
+		self.collect_garbage();
+		if time::Instant::now() >= self.next_broadcast {
+			self.rebroadcast(protocol);
+			self.next_broadcast = time::Instant::now() + REBROADCAST_INTERVAL;
+		}
+	}
+
+	/// Rebroadcast all messages to all peers.
+	fn rebroadcast(&mut self, protocol: &mut dyn Context<B>) {
+		let messages = self.messages.iter()
+			.map(|entry| (&entry.message_hash, &entry.topic, &entry.message));
+		propagate(protocol, messages, MessageIntent::PeriodicRebroadcast, &mut self.peers, &self.validators);
+	}
+
+	/// Broadcast all messages with given topic.
+	pub fn broadcast_topic(&mut self, protocol: &mut dyn Context<B>, topic: B::Hash, force: bool) {
+		let messages = self.messages.iter()
+			.filter_map(|entry|
+				if entry.topic == topic { Some((&entry.message_hash, &entry.topic, &entry.message)) } else { None }
+			);
+		let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::broadcast() };
+		propagate(protocol, messages, intent, &mut self.peers, &self.validators);
+	}
+
+	/// Prune old or no longer relevant consensus messages. Provide a predicate
+	/// for pruning, which returns `false` when the items with a given topic should be pruned.
+	pub fn collect_garbage(&mut self) {
+		self.live_message_sinks.retain(|_, sinks| {
+			sinks.retain(|sink| !sink.is_closed());
+			!sinks.is_empty()
+		});
+
+		let known_messages = &mut self.known_messages;
+		let before = self.messages.len();
+		let validators = &self.validators;
+
+		let mut check_fns = HashMap::new();
+		let mut message_expired = move |entry: &MessageEntry<B>| {
+			let engine_id = entry.message.engine_id;
+			let check_fn = match check_fns.entry(engine_id) {
+				Entry::Occupied(entry) => entry.into_mut(),
+				Entry::Vacant(vacant) => match validators.get(&engine_id) {
+					None => return true, // treat all messages with no validator as expired
+					Some(validator) => vacant.insert(validator.message_expired()),
+				}
+			};
+
+			(check_fn)(entry.topic, &entry.message.data)
+		};
+
+		self.messages.retain(|entry| !message_expired(entry));
+
+		trace!(target: "gossip", "Cleaned up {} stale messages, {} left ({} known)",
+			before - self.messages.len(),
+			self.messages.len(),
+			known_messages.len(),
+		);
+
+		for (_, ref mut peer) in self.peers.iter_mut() {
+			peer.known_messages.retain(|h| known_messages.contains_key(h));
+			peer.filtered_messages.retain(|h, _| known_messages.contains_key(h));
+		}
+	}
+
+	/// Get data of valid, incoming messages for a topic (but might have expired meanwhile)
+	pub fn messages_for(&mut self, engine_id: ConsensusEngineId, topic: B::Hash)
+		-> mpsc::UnboundedReceiver<TopicNotification>
+	{
+		let (tx, rx) = mpsc::unbounded();
+		for entry in self.messages.iter_mut()
+			.filter(|e| e.topic == topic && e.message.engine_id == engine_id)
+		{
+			tx.unbounded_send(TopicNotification {
+					message: entry.message.data.clone(),
+					sender: entry.sender.clone(),
+				})
+				.expect("receiver known to be live; qed");
+		}
+
+		self.live_message_sinks.entry((engine_id, topic)).or_default().push(tx);
+
+		rx
+	}
+
+	/// Handle an incoming ConsensusMessage for topic by who via protocol. Discard message if topic
+	/// already known, the message is old, its source peers isn't a registered peer or the connection
+	/// to them is broken. Return `Some(topic, message)` if it was added to the internal queue, `None`
+	/// in all other cases.
+	pub fn on_incoming(
+		&mut self,
+		protocol: &mut dyn Context<B>,
+		who: PeerId,
+		messages: Vec<ConsensusMessage>,
+	) {
+		trace!(target:"gossip", "Received {} messages from peer {}", messages.len(), who);
+		for message in messages {
+			let message_hash = HashFor::<B>::hash(&message.data[..]);
+
+			if self.known_messages.contains_key(&message_hash) {
+				trace!(target:"gossip", "Ignored already known message from {}", who);
+				protocol.report_peer(who.clone(), DUPLICATE_GOSSIP_REPUTATION_CHANGE);
+				continue;
+			}
+
+			let engine_id = message.engine_id;
+			// validate the message
+			let validation = self.validators.get(&engine_id)
+				.cloned()
+				.map(|v| {
+					let mut context = NetworkContext { gossip: self, protocol, engine_id };
+					v.validate(&mut context, &who, &message.data)
+				});
+
+			let validation_result = match validation {
+				Some(ValidationResult::ProcessAndKeep(topic)) => Some((topic, true)),
+				Some(ValidationResult::ProcessAndDiscard(topic)) => Some((topic, false)),
+				Some(ValidationResult::Discard) => None,
+				None => {
+					trace!(target:"gossip", "Unknown message engine id {:?} from {}", engine_id, who);
+					protocol.report_peer(who.clone(), UNKNOWN_GOSSIP_REPUTATION_CHANGE);
+					protocol.disconnect_peer(who.clone());
+					continue;
+				}
+			};
+
+			if let Some((topic, keep)) = validation_result {
+				protocol.report_peer(who.clone(), GOSSIP_SUCCESS_REPUTATION_CHANGE);
+				if let Some(ref mut peer) = self.peers.get_mut(&who) {
+					peer.known_messages.insert(message_hash);
+					if let Entry::Occupied(mut entry) = self.live_message_sinks.entry((engine_id, topic)) {
+						debug!(target: "gossip", "Pushing consensus message to sinks for {}.", topic);
+						entry.get_mut().retain(|sink| {
+							if let Err(e) = sink.unbounded_send(TopicNotification {
+								message: message.data.clone(),
+								sender: Some(who.clone())
+							}) {
+								trace!(target: "gossip", "Error broadcasting message notification: {:?}", e);
+							}
+							!sink.is_closed()
+						});
+						if entry.get().is_empty() {
+							entry.remove_entry();
+						}
+					}
+					if keep {
+						self.register_message_hashed(message_hash, topic, message, Some(who.clone()));
+					}
+				} else {
+					trace!(target:"gossip", "Ignored statement from unregistered peer {}", who);
+					protocol.report_peer(who.clone(), UNREGISTERED_TOPIC_REPUTATION_CHANGE);
+				}
+			} else {
+				trace!(target:"gossip", "Handled valid one hop message from peer {}", who);
+			}
+		}
+	}
+
+	/// Send all messages with given topic to a peer.
+	pub fn send_topic(
+		&mut self,
+		protocol: &mut dyn Context<B>,
+		who: &PeerId,
+		topic: B::Hash,
+		engine_id: ConsensusEngineId,
+		force: bool
+	) {
+		let validator = self.validators.get(&engine_id);
+		let mut message_allowed = match validator {
+			None => return, // treat all messages with no validator as not allowed
+			Some(validator) => validator.message_allowed(),
+		};
+
+		if let Some(ref mut peer) = self.peers.get_mut(who) {
+			let mut batch = Vec::new();
+			for entry in self.messages.iter().filter(|m| m.topic == topic && m.message.engine_id == engine_id) {
+				let intent = if force {
+					MessageIntent::ForcedBroadcast
+				} else {
+					let previous_attempts = peer.filtered_messages
+						.get(&entry.message_hash)
+						.cloned()
+						.unwrap_or(0);
+
+					MessageIntent::Broadcast { previous_attempts }
+				};
+
+				if !force && peer.known_messages.contains(&entry.message_hash) {
+					continue;
+				}
+
+				if !message_allowed(who, intent, &entry.topic, &entry.message.data) {
+					let count = peer.filtered_messages
+						.entry(entry.message_hash)
+						.or_insert(0);
+
+					*count += 1;
+
+					continue;
+				}
+
+				peer.filtered_messages.remove(&entry.message_hash);
+				peer.known_messages.insert(entry.message_hash.clone());
+
+				trace!(target: "gossip", "Sending topic message to {}: {:?}", who, entry.message);
+				batch.push(ConsensusMessage {
+					engine_id: engine_id.clone(),
+					data: entry.message.data.clone(),
+				});
+			}
+			protocol.send_consensus(who.clone(), batch);
+		}
+	}
+
+	/// Multicast a message to all peers.
+	pub fn multicast(
+		&mut self,
+		protocol: &mut dyn Context<B>,
+		topic: B::Hash,
+		message: ConsensusMessage,
+		force: bool,
+	) {
+		let message_hash = HashFor::<B>::hash(&message.data);
+		self.register_message_hashed(message_hash, topic, message.clone(), None);
+		let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::broadcast() };
+		propagate(protocol, iter::once((&message_hash, &topic, &message)), intent, &mut self.peers, &self.validators);
+	}
+
+	/// Send addressed message to a peer. The message is not kept or multicast
+	/// later on.
+	pub fn send_message(
+		&mut self,
+		protocol: &mut dyn Context<B>,
+		who: &PeerId,
+		message: ConsensusMessage,
+	) {
+		let peer = match self.peers.get_mut(who) {
+			None => return,
+			Some(peer) => peer,
+		};
+
+		let message_hash = HashFor::<B>::hash(&message.data);
+
+		trace!(target: "gossip", "Sending direct to {}: {:?}", who, message);
+
+		peer.filtered_messages.remove(&message_hash);
+		peer.known_messages.insert(message_hash);
+		protocol.send_consensus(who.clone(), vec![message.clone()]);
+	}
+}
+
+/// A gossip message validator that discards all messages.
+pub struct DiscardAll;
+
+impl<B: BlockT> Validator<B> for DiscardAll {
+	fn validate(
+		&self,
+		_context: &mut dyn ValidatorContext<B>,
+		_sender: &PeerId,
+		_data: &[u8],
+	) -> ValidationResult<B::Hash> {
+		ValidationResult::Discard
+	}
+
+	fn message_expired<'a>(&'a self) -> Box<dyn FnMut(B::Hash, &[u8]) -> bool + 'a> {
+		Box::new(move |_topic, _data| true)
+	}
+
+	fn message_allowed<'a>(&'a self) -> Box<dyn FnMut(&PeerId, MessageIntent, &B::Hash, &[u8]) -> bool + 'a> {
+		Box::new(move |_who, _intent, _topic, _data| false)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
+	use parking_lot::Mutex;
+	use sr_primitives::testing::{H256, Block as RawBlock, ExtrinsicWrapper};
+	use futures03::executor::block_on_stream;
+
+	use super::*;
+
+	type Block = RawBlock<ExtrinsicWrapper<u64>>;
+
+	macro_rules! push_msg {
+		($consensus:expr, $topic:expr, $hash: expr, $m:expr) => {
+			if $consensus.known_messages.insert($hash, ()).is_none() {
+				$consensus.messages.push(MessageEntry {
+					message_hash: $hash,
+					topic: $topic,
+					message: ConsensusMessage { data: $m, engine_id: [0, 0, 0, 0]},
+					sender: None,
+				});
+			}
+		}
+	}
+
+	struct AllowAll;
+	impl Validator<Block> for AllowAll {
+		fn validate(
+			&self,
+			_context: &mut dyn ValidatorContext<Block>,
+			_sender: &PeerId,
+			_data: &[u8],
+		) -> ValidationResult<H256> {
+			ValidationResult::ProcessAndKeep(H256::default())
+		}
+	}
+
+	#[test]
+	fn collects_garbage() {
+		struct AllowOne;
+		impl Validator<Block> for AllowOne {
+			fn validate(
+				&self,
+				_context: &mut dyn ValidatorContext<Block>,
+				_sender: &PeerId,
+				data: &[u8],
+			) -> ValidationResult<H256> {
+				if data[0] == 1 {
+					ValidationResult::ProcessAndKeep(H256::default())
+				} else {
+					ValidationResult::Discard
+				}
+			}
+
+			fn message_expired<'a>(&'a self) -> Box<dyn FnMut(H256, &[u8]) -> bool + 'a> {
+				Box::new(move |_topic, data| data[0] != 1)
+			}
+		}
+
+		let prev_hash = H256::random();
+		let best_hash = H256::random();
+		let mut consensus = ConsensusGossip::<Block>::new();
+		let m1_hash = H256::random();
+		let m2_hash = H256::random();
+		let m1 = vec![1, 2, 3];
+		let m2 = vec![4, 5, 6];
+
+		push_msg!(consensus, prev_hash, m1_hash, m1);
+		push_msg!(consensus, best_hash, m2_hash, m2);
+		consensus.known_messages.insert(m1_hash, ());
+		consensus.known_messages.insert(m2_hash, ());
+
+		let test_engine_id = Default::default();
+		consensus.register_validator_internal(test_engine_id, Arc::new(AllowAll));
+		consensus.collect_garbage();
+		assert_eq!(consensus.messages.len(), 2);
+		assert_eq!(consensus.known_messages.len(), 2);
+
+		consensus.register_validator_internal(test_engine_id, Arc::new(AllowOne));
+
+		// m2 is expired
+		consensus.collect_garbage();
+		assert_eq!(consensus.messages.len(), 1);
+		// known messages are only pruned based on size.
+		assert_eq!(consensus.known_messages.len(), 2);
+		assert!(consensus.known_messages.contains_key(&m2_hash));
+	}
+
+	#[test]
+	fn message_stream_include_those_sent_before_asking_for_stream() {
+		let mut consensus = ConsensusGossip::<Block>::new();
+		consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll));
+
+		let message = ConsensusMessage { data: vec![4, 5, 6], engine_id: [0, 0, 0, 0] };
+		let topic = HashFor::<Block>::hash(&[1,2,3]);
+
+		consensus.register_message(topic, message.clone());
+		let mut stream = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic));
+
+		assert_eq!(stream.next(), Some(TopicNotification { message: message.data, sender: None }));
+	}
+
+	#[test]
+	fn can_keep_multiple_messages_per_topic() {
+		let mut consensus = ConsensusGossip::<Block>::new();
+
+		let topic = [1; 32].into();
+		let msg_a = ConsensusMessage { data: vec![1, 2, 3], engine_id: [0, 0, 0, 0] };
+		let msg_b = ConsensusMessage { data: vec![4, 5, 6], engine_id: [0, 0, 0, 0] };
+
+		consensus.register_message(topic, msg_a);
+		consensus.register_message(topic, msg_b);
+
+		assert_eq!(consensus.messages.len(), 2);
+	}
+
+	#[test]
+	fn can_keep_multiple_subscribers_per_topic() {
+		let mut consensus = ConsensusGossip::<Block>::new();
+		consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll));
+
+		let data = vec![4, 5, 6];
+		let message = ConsensusMessage { data: data.clone(), engine_id: [0, 0, 0, 0] };
+		let topic = HashFor::<Block>::hash(&[1, 2, 3]);
+
+		consensus.register_message(topic, message.clone());
+
+		let mut stream1 = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic));
+		let mut stream2 = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic));
+
+		assert_eq!(stream1.next(), Some(TopicNotification { message: data.clone(), sender: None }));
+		assert_eq!(stream2.next(), Some(TopicNotification { message: data, sender: None }));
+	}
+
+	#[test]
+	fn topics_are_localized_to_engine_id() {
+		let mut consensus = ConsensusGossip::<Block>::new();
+		consensus.register_validator_internal([0, 0, 0, 0], Arc::new(AllowAll));
+
+		let topic = [1; 32].into();
+		let msg_a = ConsensusMessage { data: vec![1, 2, 3], engine_id: [0, 0, 0, 0] };
+		let msg_b = ConsensusMessage { data: vec![4, 5, 6], engine_id: [0, 0, 0, 1] };
+
+		consensus.register_message(topic, msg_a);
+		consensus.register_message(topic, msg_b);
+
+		let mut stream = block_on_stream(consensus.messages_for([0, 0, 0, 0], topic));
+
+		assert_eq!(stream.next(), Some(TopicNotification { message: vec![1, 2, 3], sender: None }));
+
+		let _ = consensus.live_message_sinks.remove(&([0, 0, 0, 0], topic));
+		assert_eq!(stream.next(), None);
+	}
+
+	#[test]
+	fn keeps_track_of_broadcast_attempts() {
+		struct DummyNetworkContext;
+		impl<B: BlockT> Context<B> for DummyNetworkContext {
+			fn report_peer(&mut self, _who: PeerId, _reputation: i32) {}
+			fn disconnect_peer(&mut self, _who: PeerId) {}
+			fn send_consensus(&mut self, _who: PeerId, _consensus: Vec<ConsensusMessage>) {}
+			fn send_chain_specific(&mut self, _who: PeerId, _message: Vec<u8>) {}
+		}
+
+		// A mock gossip validator that never expires any message, allows
+		// setting whether messages should be allowed and keeps track of any
+		// messages passed to `message_allowed`.
+		struct MockValidator {
+			allow: AtomicBool,
+			messages: Arc<Mutex<Vec<(Vec<u8>, MessageIntent)>>>,
+		}
+
+		impl MockValidator {
+			fn new() -> MockValidator {
+				MockValidator {
+					allow: AtomicBool::new(false),
+					messages: Arc::new(Mutex::new(Vec::new())),
+				}
+			}
+		}
+
+		impl Validator<Block> for MockValidator {
+			fn validate(
+				&self,
+				_context: &mut dyn ValidatorContext<Block>,
+				_sender: &PeerId,
+				_data: &[u8],
+			) -> ValidationResult<H256> {
+				ValidationResult::ProcessAndKeep(H256::default())
+			}
+
+			fn message_expired<'a>(&'a self) -> Box<dyn FnMut(H256, &[u8]) -> bool + 'a> {
+				Box::new(move |_topic, _data| false)
+			}
+
+			fn message_allowed<'a>(&'a self) -> Box<dyn FnMut(&PeerId, MessageIntent, &H256, &[u8]) -> bool + 'a> {
+				let messages = self.messages.clone();
+				Box::new(move |_, intent, _, data| {
+					messages.lock().push((data.to_vec(), intent));
+					self.allow.load(Ordering::SeqCst)
+				})
+			}
+		}
+
+		// we setup an instance of the mock gossip validator, add a new peer to
+		// it and register a message.
+		let mut consensus = ConsensusGossip::<Block>::new();
+		let validator = Arc::new(MockValidator::new());
+		consensus.register_validator_internal([0, 0, 0, 0], validator.clone());
+		consensus.new_peer(
+			&mut DummyNetworkContext,
+			PeerId::random(),
+			Roles::AUTHORITY,
+		);
+
+		let data = vec![1, 2, 3];
+		let msg = ConsensusMessage { data: data.clone(), engine_id: [0, 0, 0, 0] };
+		consensus.register_message(H256::default(), msg);
+
+		// tick the gossip handler and make sure it triggers a message rebroadcast
+		let mut tick = || {
+			consensus.next_broadcast = std::time::Instant::now();
+			consensus.tick(&mut DummyNetworkContext);
+		};
+
+		// by default we won't allow the message we registered, so everytime we
+		// tick the gossip handler, the message intent should be kept as
+		// `Broadcast` but the previous attempts should be incremented.
+		tick();
+		assert_eq!(
+			validator.messages.lock().pop().unwrap(),
+			(data.clone(), MessageIntent::Broadcast { previous_attempts: 0 }),
+		);
+
+		tick();
+		assert_eq!(
+			validator.messages.lock().pop().unwrap(),
+			(data.clone(), MessageIntent::Broadcast { previous_attempts: 1 }),
+		);
+
+		// we set the validator to allow the message to go through
+		validator.allow.store(true, Ordering::SeqCst);
+
+		// we still get the same message intent but it should be delivered now
+		tick();
+		assert_eq!(
+			validator.messages.lock().pop().unwrap(),
+			(data.clone(), MessageIntent::Broadcast { previous_attempts: 2 }),
+		);
+
+		// ticking the gossip handler again the message intent should change to
+		// `PeriodicRebroadcast` since it was sent.
+		tick();
+		assert_eq!(
+			validator.messages.lock().pop().unwrap(),
+			(data.clone(), MessageIntent::PeriodicRebroadcast),
+		);
+	}
+}

substrate/client/network/src/protocol/event.rs

@@ -0,0 +1,43 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Network event types. These are are not the part of the protocol, but rather
+//! events that happen on the network like DHT get/put results received.
+
+use libp2p::kad::record::Key;
+
+/// Events generated by DHT as a response to get_value and put_value requests.
+#[derive(Debug, Clone)]
+pub enum DhtEvent {
+	/// The value was found.
+	ValueFound(Vec<(Key, Vec<u8>)>),
+
+	/// The requested record has not been found in the DHT.
+	ValueNotFound(Key),
+
+	/// The record has been successfully inserted into the DHT.
+	ValuePut(Key),
+
+	/// An error has occured while putting a record into the DHT.
+	ValuePutFailed(Key),
+}
+
+/// Type for events generated by networking layer.
+#[derive(Debug, Clone)]
+pub enum Event {
+	/// Event generated by a DHT.
+	Dht(DhtEvent),
+}

substrate/client/network/src/protocol/message.rs

@@ -0,0 +1,452 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Network packet message types. These get serialized and put into the lower level protocol payload.
+
+use bitflags::bitflags;
+use sr_primitives::{ConsensusEngineId, traits::{Block as BlockT, Header as HeaderT}};
+use codec::{Encode, Decode, Input, Output, Error};
+pub use self::generic::{
+	BlockAnnounce, RemoteCallRequest, RemoteReadRequest,
+	RemoteHeaderRequest, RemoteHeaderResponse,
+	RemoteChangesRequest, RemoteChangesResponse,
+	FinalityProofRequest, FinalityProofResponse,
+	FromBlock, RemoteReadChildRequest,
+};
+use client_api::StorageProof;
+
+/// A unique ID of a request.
+pub type RequestId = u64;
+
+/// Type alias for using the message type using block type parameters.
+pub type Message<B> = generic::Message<
+	<B as BlockT>::Header,
+	<B as BlockT>::Hash,
+	<<B as BlockT>::Header as HeaderT>::Number,
+	<B as BlockT>::Extrinsic,
+>;
+
+/// Type alias for using the status type using block type parameters.
+pub type Status<B> = generic::Status<
+	<B as BlockT>::Hash,
+	<<B as BlockT>::Header as HeaderT>::Number,
+>;
+
+/// Type alias for using the block request type using block type parameters.
+pub type BlockRequest<B> = generic::BlockRequest<
+	<B as BlockT>::Hash,
+	<<B as BlockT>::Header as HeaderT>::Number,
+>;
+
+/// Type alias for using the BlockData type using block type parameters.
+pub type BlockData<B> = generic::BlockData<
+	<B as BlockT>::Header,
+	<B as BlockT>::Hash,
+	<B as BlockT>::Extrinsic,
+>;
+
+/// Type alias for using the BlockResponse type using block type parameters.
+pub type BlockResponse<B> = generic::BlockResponse<
+	<B as BlockT>::Header,
+	<B as BlockT>::Hash,
+	<B as BlockT>::Extrinsic,
+>;
+
+/// A set of transactions.
+pub type Transactions<E> = Vec<E>;
+
+// Bits of block data and associated artifacts to request.
+bitflags! {
+	/// Node roles bitmask.
+	pub struct BlockAttributes: u8 {
+		/// Include block header.
+		const HEADER = 0b00000001;
+		/// Include block body.
+		const BODY = 0b00000010;
+		/// Include block receipt.
+		const RECEIPT = 0b00000100;
+		/// Include block message queue.
+		const MESSAGE_QUEUE = 0b00001000;
+		/// Include a justification for the block.
+		const JUSTIFICATION = 0b00010000;
+	}
+}
+
+impl Encode for BlockAttributes {
+	fn encode_to<T: Output>(&self, dest: &mut T) {
+		dest.push_byte(self.bits())
+	}
+}
+
+impl codec::EncodeLike for BlockAttributes {}
+
+impl Decode for BlockAttributes {
+	fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
+		Self::from_bits(input.read_byte()?).ok_or_else(|| Error::from("Invalid bytes"))
+	}
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Encode, Decode)]
+/// Block enumeration direction.
+pub enum Direction {
+	/// Enumerate in ascending order (from child to parent).
+	Ascending = 0,
+	/// Enumerate in descending order (from parent to canonical child).
+	Descending = 1,
+}
+
+/// Block state in the chain.
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Encode, Decode)]
+pub enum BlockState {
+	/// Block is not part of the best chain.
+	Normal,
+	/// Latest best block.
+	Best,
+}
+
+/// Remote call response.
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+pub struct RemoteCallResponse {
+	/// Id of a request this response was made for.
+	pub id: RequestId,
+	/// Execution proof.
+	pub proof: StorageProof,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+/// Remote read response.
+pub struct RemoteReadResponse {
+	/// Id of a request this response was made for.
+	pub id: RequestId,
+	/// Read proof.
+	pub proof: StorageProof,
+}
+
+/// Generic types.
+pub mod generic {
+	use codec::{Encode, Decode, Input, Output};
+	use sr_primitives::Justification;
+	use crate::config::Roles;
+	use super::{
+		RemoteReadResponse, Transactions, Direction,
+		RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId,
+		BlockState, StorageProof,
+	};
+	/// Consensus is mostly opaque to us
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub struct ConsensusMessage {
+		/// Identifies consensus engine.
+		pub engine_id: ConsensusEngineId,
+		/// Message payload.
+		pub data: Vec<u8>,
+	}
+
+	/// Block data sent in the response.
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub struct BlockData<Header, Hash, Extrinsic> {
+		/// Block header hash.
+		pub hash: Hash,
+		/// Block header if requested.
+		pub header: Option<Header>,
+		/// Block body if requested.
+		pub body: Option<Vec<Extrinsic>>,
+		/// Block receipt if requested.
+		pub receipt: Option<Vec<u8>>,
+		/// Block message queue if requested.
+		pub message_queue: Option<Vec<u8>>,
+		/// Justification if requested.
+		pub justification: Option<Justification>,
+	}
+
+	/// Identifies starting point of a block sequence.
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub enum FromBlock<Hash, Number> {
+		/// Start with given hash.
+		Hash(Hash),
+		/// Start with given block number.
+		Number(Number),
+	}
+
+	/// A network message.
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub enum Message<Header, Hash, Number, Extrinsic> {
+		/// Status packet.
+		Status(Status<Hash, Number>),
+		/// Block request.
+		BlockRequest(BlockRequest<Hash, Number>),
+		/// Block response.
+		BlockResponse(BlockResponse<Header, Hash, Extrinsic>),
+		/// Block announce.
+		BlockAnnounce(BlockAnnounce<Header>),
+		/// Transactions.
+		Transactions(Transactions<Extrinsic>),
+		/// Consensus protocol message.
+		Consensus(ConsensusMessage),
+		/// Remote method call request.
+		RemoteCallRequest(RemoteCallRequest<Hash>),
+		/// Remote method call response.
+		RemoteCallResponse(RemoteCallResponse),
+		/// Remote storage read request.
+		RemoteReadRequest(RemoteReadRequest<Hash>),
+		/// Remote storage read response.
+		RemoteReadResponse(RemoteReadResponse),
+		/// Remote header request.
+		RemoteHeaderRequest(RemoteHeaderRequest<Number>),
+		/// Remote header response.
+		RemoteHeaderResponse(RemoteHeaderResponse<Header>),
+		/// Remote changes request.
+		RemoteChangesRequest(RemoteChangesRequest<Hash>),
+		/// Remote changes reponse.
+		RemoteChangesResponse(RemoteChangesResponse<Number, Hash>),
+		/// Remote child storage read request.
+		RemoteReadChildRequest(RemoteReadChildRequest<Hash>),
+		/// Finality proof request.
+		FinalityProofRequest(FinalityProofRequest<Hash>),
+		/// Finality proof reponse.
+		FinalityProofResponse(FinalityProofResponse<Hash>),
+		/// Batch of consensus protocol messages.
+		ConsensusBatch(Vec<ConsensusMessage>),
+		/// Chain-specific message.
+		#[codec(index = "255")]
+		ChainSpecific(Vec<u8>),
+	}
+
+	impl<Header, Hash, Number, Extrinsic> Message<Header, Hash, Number, Extrinsic> {
+		/// Message id useful for logging.
+		pub fn id(&self) -> &'static str {
+			match self {
+				Message::Status(_) => "Status",
+				Message::BlockRequest(_) => "BlockRequest",
+				Message::BlockResponse(_) => "BlockResponse",
+				Message::BlockAnnounce(_) => "BlockAnnounce",
+				Message::Transactions(_) => "Transactions",
+				Message::Consensus(_) => "Consensus",
+				Message::RemoteCallRequest(_) => "RemoteCallRequest",
+				Message::RemoteCallResponse(_) => "RemoteCallResponse",
+				Message::RemoteReadRequest(_) => "RemoteReadRequest",
+				Message::RemoteReadResponse(_) => "RemoteReadResponse",
+				Message::RemoteHeaderRequest(_) => "RemoteHeaderRequest",
+				Message::RemoteHeaderResponse(_) => "RemoteHeaderResponse",
+				Message::RemoteChangesRequest(_) => "RemoteChangesRequest",
+				Message::RemoteChangesResponse(_) => "RemoteChangesResponse",
+				Message::RemoteReadChildRequest(_) => "RemoteReadChildRequest",
+				Message::FinalityProofRequest(_) => "FinalityProofRequest",
+				Message::FinalityProofResponse(_) => "FinalityProofResponse",
+				Message::ConsensusBatch(_) => "ConsensusBatch",
+				Message::ChainSpecific(_) => "ChainSpecific",
+			}
+		}
+	}
+
+	/// Status sent on connection.
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub struct Status<Hash, Number> {
+		/// Protocol version.
+		pub version: u32,
+		/// Minimum supported version.
+		pub min_supported_version: u32,
+		/// Supported roles.
+		pub roles: Roles,
+		/// Best block number.
+		pub best_number: Number,
+		/// Best block hash.
+		pub best_hash: Hash,
+		/// Genesis block hash.
+		pub genesis_hash: Hash,
+		/// Chain-specific status.
+		pub chain_status: Vec<u8>,
+	}
+
+	/// Request block data from a peer.
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub struct BlockRequest<Hash, Number> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Bits of block data to request.
+		pub fields: BlockAttributes,
+		/// Start from this block.
+		pub from: FromBlock<Hash, Number>,
+		/// End at this block. An implementation defined maximum is used when unspecified.
+		pub to: Option<Hash>,
+		/// Sequence direction.
+		pub direction: Direction,
+		/// Maximum number of blocks to return. An implementation defined maximum is used when unspecified.
+		pub max: Option<u32>,
+	}
+
+	/// Response to `BlockRequest`
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	pub struct BlockResponse<Header, Hash, Extrinsic> {
+		/// Id of a request this response was made for.
+		pub id: RequestId,
+		/// Block data for the requested sequence.
+		pub blocks: Vec<BlockData<Header, Hash, Extrinsic>>,
+	}
+
+	/// Announce a new complete relay chain block on the network.
+	#[derive(Debug, PartialEq, Eq, Clone)]
+	pub struct BlockAnnounce<H> {
+		/// New block header.
+		pub header: H,
+		/// Block state. TODO: Remove `Option` and custom encoding when v4 becomes common.
+		pub state: Option<BlockState>,
+		/// Data associated with this block announcement, e.g. a candidate message.
+		pub data: Option<Vec<u8>>,
+	}
+
+	// Custom Encode/Decode impl to maintain backwards compatibility with v3.
+	// This assumes that the packet contains nothing but the announcement message.
+	// TODO: Get rid of it once protocol v4 is common.
+	impl<H: Encode> Encode for BlockAnnounce<H> {
+		fn encode_to<T: Output>(&self, dest: &mut T) {
+			self.header.encode_to(dest);
+			if let Some(state) = &self.state {
+				state.encode_to(dest);
+			}
+			if let Some(data) = &self.data {
+				data.encode_to(dest)
+			}
+		}
+	}
+
+	impl<H: Decode> Decode for BlockAnnounce<H> {
+		fn decode<I: Input>(input: &mut I) -> Result<Self, codec::Error> {
+			let header = H::decode(input)?;
+			let state = BlockState::decode(input).ok();
+			let data = Vec::decode(input).ok();
+			Ok(BlockAnnounce {
+				header,
+				state,
+				data,
+			})
+		}
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote call request.
+	pub struct RemoteCallRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Block at which to perform call.
+		pub block: H,
+		/// Method name.
+		pub method: String,
+		/// Call data.
+		pub data: Vec<u8>,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote storage read request.
+	pub struct RemoteReadRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Block at which to perform call.
+		pub block: H,
+		/// Storage key.
+		pub keys: Vec<Vec<u8>>,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote storage read child request.
+	pub struct RemoteReadChildRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Block at which to perform call.
+		pub block: H,
+		/// Child Storage key.
+		pub storage_key: Vec<u8>,
+		/// Storage key.
+		pub keys: Vec<Vec<u8>>,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote header request.
+	pub struct RemoteHeaderRequest<N> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Block number to request header for.
+		pub block: N,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote header response.
+	pub struct RemoteHeaderResponse<Header> {
+		/// Id of a request this response was made for.
+		pub id: RequestId,
+		/// Header. None if proof generation has failed (e.g. header is unknown).
+		pub header: Option<Header>,
+		/// Header proof.
+		pub proof: StorageProof,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote changes request.
+	pub struct RemoteChangesRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Hash of the first block of the range (including first) where changes are requested.
+		pub first: H,
+		/// Hash of the last block of the range (including last) where changes are requested.
+		pub last: H,
+		/// Hash of the first block for which the requester has the changes trie root. All other
+		/// affected roots must be proved.
+		pub min: H,
+		/// Hash of the last block that we can use when querying changes.
+		pub max: H,
+		/// Storage child node key which changes are requested.
+		pub storage_key: Option<Vec<u8>>,
+		/// Storage key which changes are requested.
+		pub key: Vec<u8>,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Remote changes response.
+	pub struct RemoteChangesResponse<N, H> {
+		/// Id of a request this response was made for.
+		pub id: RequestId,
+		/// Proof has been generated using block with this number as a max block. Should be
+		/// less than or equal to the RemoteChangesRequest::max block number.
+		pub max: N,
+		/// Changes proof.
+		pub proof: Vec<Vec<u8>>,
+		/// Changes tries roots missing on the requester' node.
+		pub roots: Vec<(N, H)>,
+		/// Missing changes tries roots proof.
+		pub roots_proof: StorageProof,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Finality proof request.
+	pub struct FinalityProofRequest<H> {
+		/// Unique request id.
+		pub id: RequestId,
+		/// Hash of the block to request proof for.
+		pub block: H,
+		/// Additional data blob (that both requester and provider understood) required for proving finality.
+		pub request: Vec<u8>,
+	}
+
+	#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)]
+	/// Finality proof response.
+	pub struct FinalityProofResponse<H> {
+		/// Id of a request this response was made for.
+		pub id: RequestId,
+		/// Hash of the block (the same as in the FinalityProofRequest).
+		pub block: H,
+		/// Finality proof (if available).
+		pub proof: Option<Vec<u8>>,
+	}
+}

substrate/client/network/src/protocol/specialization.rs

@@ -0,0 +1,164 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Specializations of the substrate network protocol to allow more complex forms of communication.
+
+pub use crate::protocol::event::{DhtEvent, Event};
+
+use crate::protocol::Context;
+use libp2p::PeerId;
+use sr_primitives::traits::Block as BlockT;
+
+/// A specialization of the substrate network protocol. Handles events and sends messages.
+pub trait NetworkSpecialization<B: BlockT>: Send + Sync + 'static {
+	/// Get the current specialization-status.
+	fn status(&self) -> Vec<u8>;
+
+	/// Called when a peer successfully handshakes.
+	fn on_connect(&mut self, ctx: &mut dyn Context<B>, who: PeerId, status: crate::message::Status<B>);
+
+	/// Called when a peer is disconnected. If the peer ID is unknown, it should be ignored.
+	fn on_disconnect(&mut self, ctx: &mut dyn Context<B>, who: PeerId);
+
+	/// Called when a network-specific message arrives.
+	fn on_message(
+		&mut self,
+		ctx: &mut dyn Context<B>,
+		who: PeerId,
+		message: Vec<u8>
+	);
+
+	/// Called when a network-specific event arrives.
+	#[deprecated(note = "This method is never called; please use `with_dht_event_tx` when building the service")]
+	fn on_event(&mut self, _event: Event) {}
+
+	/// Called on abort.
+	#[deprecated(note = "This method is never called; aborting corresponds to dropping the object")]
+	fn on_abort(&mut self) { }
+
+	/// Called periodically to maintain peers and handle timeouts.
+	fn maintain_peers(&mut self, _ctx: &mut dyn Context<B>) { }
+
+	/// Called when a block is _imported_ at the head of the chain (not during major sync).
+	/// Not guaranteed to be called for every block, but will be most of the after major sync.
+	fn on_block_imported(&mut self, _ctx: &mut dyn Context<B>, _hash: B::Hash, _header: &B::Header) { }
+}
+
+/// Construct a simple protocol that is composed of several sub protocols.
+/// Each "sub protocol" needs to implement `Specialization` and needs to provide a `new()` function.
+/// For more fine grained implementations, this macro is not usable.
+///
+/// # Example
+///
+/// ```nocompile
+/// construct_simple_protocol! {
+///     pub struct MyProtocol where Block = MyBlock {
+///         consensus_gossip: ConsensusGossip<MyBlock>,
+///         other_protocol: MyCoolStuff,
+///     }
+/// }
+/// ```
+///
+/// You can also provide an optional parameter after `where Block = MyBlock`, so it looks like
+/// `where Block = MyBlock, Status = consensus_gossip`. This will instruct the implementation to
+/// use the `status()` function from the `ConsensusGossip` protocol. By default, `status()` returns
+/// an empty vector.
+#[macro_export]
+macro_rules! construct_simple_protocol {
+	(
+		$( #[ $attr:meta ] )*
+		pub struct $protocol:ident where
+			Block = $block:ident
+			$( , Status = $status_protocol_name:ident )*
+		{
+			$( $sub_protocol_name:ident : $sub_protocol:ident $( <$protocol_block:ty> )*, )*
+		}
+	) => {
+		$( #[$attr] )*
+		pub struct $protocol {
+			$( $sub_protocol_name: $sub_protocol $( <$protocol_block> )*, )*
+		}
+
+		impl $protocol {
+			/// Instantiate a node protocol handler.
+			pub fn new() -> Self {
+				Self {
+					$( $sub_protocol_name: $sub_protocol::new(), )*
+				}
+			}
+		}
+
+		impl $crate::specialization::NetworkSpecialization<$block> for $protocol {
+			fn status(&self) -> Vec<u8> {
+				$(
+					let status = self.$status_protocol_name.status();
+
+					if !status.is_empty() {
+						return status;
+					}
+				)*
+
+				Vec::new()
+			}
+
+			fn on_connect(
+				&mut self,
+				_ctx: &mut $crate::Context<$block>,
+				_who: $crate::PeerId,
+				_status: $crate::StatusMessage<$block>
+			) {
+				$( self.$sub_protocol_name.on_connect(_ctx, _who, _status); )*
+			}
+
+			fn on_disconnect(&mut self, _ctx: &mut $crate::Context<$block>, _who: $crate::PeerId) {
+				$( self.$sub_protocol_name.on_disconnect(_ctx, _who); )*
+			}
+
+			fn on_message(
+				&mut self,
+				_ctx: &mut $crate::Context<$block>,
+				_who: $crate::PeerId,
+				_message: Vec<u8>,
+			) {
+				$( self.$sub_protocol_name.on_message(_ctx, _who, _message); )*
+			}
+
+			fn on_event(
+				&mut self,
+				_event: $crate::specialization::Event
+			) {
+				$( self.$sub_protocol_name.on_event(_event); )*
+			}
+
+			fn on_abort(&mut self) {
+				$( self.$sub_protocol_name.on_abort(); )*
+			}
+
+			fn maintain_peers(&mut self, _ctx: &mut $crate::Context<$block>) {
+				$( self.$sub_protocol_name.maintain_peers(_ctx); )*
+			}
+
+			fn on_block_imported(
+				&mut self,
+				_ctx: &mut $crate::Context<$block>,
+				_hash: <$block as $crate::BlockT>::Hash,
+				_header: &<$block as $crate::BlockT>::Header
+			) {
+				$( self.$sub_protocol_name.on_block_imported(_ctx, _hash, _header); )*
+			}
+		}
+	}
+}

substrate/client/network/src/protocol/sync/blocks.rs

@@ -0,0 +1,312 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::mem;
+use std::cmp;
+use std::ops::Range;
+use std::collections::{HashMap, BTreeMap};
+use std::collections::hash_map::Entry;
+use log::trace;
+use libp2p::PeerId;
+use sr_primitives::traits::{Block as BlockT, NumberFor, One};
+use crate::message;
+
+/// Block data with origin.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlockData<B: BlockT> {
+	/// The Block Message from the wire
+	pub block: message::BlockData<B>,
+	/// The peer, we received this from
+	pub origin: Option<PeerId>,
+}
+
+#[derive(Debug)]
+enum BlockRangeState<B: BlockT> {
+	Downloading {
+		len: NumberFor<B>,
+		downloading: u32,
+	},
+	Complete(Vec<BlockData<B>>),
+}
+
+impl<B: BlockT> BlockRangeState<B> {
+	pub fn len(&self) -> NumberFor<B> {
+		match *self {
+			BlockRangeState::Downloading { len, .. } => len,
+			BlockRangeState::Complete(ref blocks) => (blocks.len() as u32).into(),
+		}
+	}
+}
+
+/// A collection of blocks being downloaded.
+#[derive(Default)]
+pub struct BlockCollection<B: BlockT> {
+	/// Downloaded blocks.
+	blocks: BTreeMap<NumberFor<B>, BlockRangeState<B>>,
+	peer_requests: HashMap<PeerId, NumberFor<B>>,
+}
+
+impl<B: BlockT> BlockCollection<B> {
+	/// Create a new instance.
+	pub fn new() -> Self {
+		BlockCollection {
+			blocks: BTreeMap::new(),
+			peer_requests: HashMap::new(),
+		}
+	}
+
+	/// Clear everything.
+	pub fn clear(&mut self) {
+		self.blocks.clear();
+		self.peer_requests.clear();
+	}
+
+	/// Insert a set of blocks into collection.
+	pub fn insert(&mut self, start: NumberFor<B>, blocks: Vec<message::BlockData<B>>, who: PeerId) {
+		if blocks.is_empty() {
+			return;
+		}
+
+		match self.blocks.get(&start) {
+			Some(&BlockRangeState::Downloading { .. }) => {
+				trace!(target: "sync", "Inserting block data still marked as being downloaded: {}", start);
+			},
+			Some(&BlockRangeState::Complete(ref existing)) if existing.len() >= blocks.len() => {
+				trace!(target: "sync", "Ignored block data already downloaded: {}", start);
+				return;
+			},
+			_ => (),
+		}
+
+		self.blocks.insert(start, BlockRangeState::Complete(blocks.into_iter()
+			.map(|b| BlockData { origin: Some(who.clone()), block: b }).collect()));
+	}
+
+	/// Returns a set of block hashes that require a header download. The returned set is marked as being downloaded.
+	pub fn needed_blocks(
+		&mut self,
+		who: PeerId,
+		count: usize,
+		peer_best: NumberFor<B>,
+		common: NumberFor<B>,
+		max_parallel: u32,
+		max_ahead: u32,
+	) -> Option<Range<NumberFor<B>>>
+	{
+		if peer_best <= common {
+			// Bail out early
+			return None;
+		}
+		// First block number that we need to download
+		let first_different = common + <NumberFor<B>>::one();
+		let count = (count as u32).into();
+		let (mut range, downloading) = {
+			let mut downloading_iter = self.blocks.iter().peekable();
+			let mut prev: Option<(&NumberFor<B>, &BlockRangeState<B>)> = None;
+			loop {
+				let next = downloading_iter.next();
+				break match &(prev, next) {
+					&(Some((start, &BlockRangeState::Downloading { ref len, downloading })), _)
+						if downloading < max_parallel =>
+						(*start .. *start + *len, downloading),
+					&(Some((start, r)), Some((next_start, _))) if *start + r.len() < *next_start =>
+						(*start + r.len() .. cmp::min(*next_start, *start + r.len() + count), 0), // gap
+					&(Some((start, r)), None) =>
+						(*start + r.len() .. *start + r.len() + count, 0), // last range
+					&(None, None) =>
+						(first_different .. first_different + count, 0), // empty
+					&(None, Some((start, _))) if *start > first_different =>
+						(first_different .. cmp::min(first_different + count, *start), 0), // gap at the start
+					_ => {
+						prev = next;
+						continue
+					},
+				}
+			}
+		};
+		// crop to peers best
+		if range.start > peer_best {
+			trace!(target: "sync", "Out of range for peer {} ({} vs {})", who, range.start, peer_best);
+			return None;
+		}
+		range.end = cmp::min(peer_best + One::one(), range.end);
+
+		if self.blocks.iter().next().map_or(false, |(n, _)| range.start > *n + max_ahead.into()) {
+			trace!(target: "sync", "Too far ahead for peer {} ({})", who, range.start);
+			return None;
+		}
+
+		self.peer_requests.insert(who, range.start);
+		self.blocks.insert(range.start, BlockRangeState::Downloading {
+			len: range.end - range.start,
+			downloading: downloading + 1
+		});
+		if range.end <= range.start {
+			panic!("Empty range {:?}, count={}, peer_best={}, common={}, blocks={:?}",
+				range, count, peer_best, common, self.blocks);
+		}
+		Some(range)
+	}
+
+	/// Get a valid chain of blocks ordered in descending order and ready for importing into blockchain.
+	pub fn drain(&mut self, from: NumberFor<B>) -> Vec<BlockData<B>> {
+		let mut drained = Vec::new();
+		let mut ranges = Vec::new();
+		{
+			let mut prev = from;
+			for (start, range_data) in &mut self.blocks {
+				match range_data {
+					&mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => {
+							prev = *start + (blocks.len() as u32).into();
+							let mut blocks = mem::replace(blocks, Vec::new());
+							drained.append(&mut blocks);
+							ranges.push(*start);
+					},
+					_ => break,
+				}
+			}
+		}
+		for r in ranges {
+			self.blocks.remove(&r);
+		}
+		trace!(target: "sync", "Drained {} blocks", drained.len());
+		drained
+	}
+
+	pub fn clear_peer_download(&mut self, who: &PeerId) {
+		match self.peer_requests.entry(who.clone()) {
+			Entry::Occupied(entry) => {
+				let start = entry.remove();
+				let remove = match self.blocks.get_mut(&start) {
+					Some(&mut BlockRangeState::Downloading { ref mut downloading, .. }) if *downloading > 1 => {
+						*downloading = *downloading - 1;
+						false
+					},
+					Some(&mut BlockRangeState::Downloading { .. }) => {
+						true
+					},
+					_ => {
+						false
+					}
+				};
+				if remove {
+					self.blocks.remove(&start);
+				}
+			},
+			_ => (),
+		}
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use super::{BlockCollection, BlockData, BlockRangeState};
+	use crate::{message, PeerId};
+	use sr_primitives::testing::{Block as RawBlock, ExtrinsicWrapper};
+	use primitives::H256;
+
+	type Block = RawBlock<ExtrinsicWrapper<u64>>;
+
+	fn is_empty(bc: &BlockCollection<Block>) -> bool {
+		bc.blocks.is_empty() &&
+		bc.peer_requests.is_empty()
+	}
+
+	fn generate_blocks(n: usize) -> Vec<message::BlockData<Block>> {
+		(0 .. n).map(|_| message::generic::BlockData {
+			hash: H256::random(),
+			header: None,
+			body: None,
+			message_queue: None,
+			receipt: None,
+			justification: None,
+		}).collect()
+	}
+
+	#[test]
+	fn create_clear() {
+		let mut bc = BlockCollection::new();
+		assert!(is_empty(&bc));
+		bc.insert(1, generate_blocks(100), PeerId::random());
+		assert!(!is_empty(&bc));
+		bc.clear();
+		assert!(is_empty(&bc));
+	}
+
+	#[test]
+	fn insert_blocks() {
+		let mut bc = BlockCollection::new();
+		assert!(is_empty(&bc));
+		let peer0 = PeerId::random();
+		let peer1 = PeerId::random();
+		let peer2 = PeerId::random();
+
+		let blocks = generate_blocks(150);
+		assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(1 .. 41));
+		assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(41 .. 81));
+		assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 0, 1, 200), Some(81 .. 121));
+
+		bc.clear_peer_download(&peer1);
+		bc.insert(41, blocks[41..81].to_vec(), peer1.clone());
+		assert_eq!(bc.drain(1), vec![]);
+		assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(121 .. 151));
+		bc.clear_peer_download(&peer0);
+		bc.insert(1, blocks[1..11].to_vec(), peer0.clone());
+
+		assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(11 .. 41));
+		assert_eq!(bc.drain(1), blocks[1..11].iter()
+			.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>());
+
+		bc.clear_peer_download(&peer0);
+		bc.insert(11, blocks[11..41].to_vec(), peer0.clone());
+
+		let drained = bc.drain(12);
+		assert_eq!(drained[..30], blocks[11..41].iter()
+			.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>()[..]);
+		assert_eq!(drained[30..], blocks[41..81].iter()
+			.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
+
+		bc.clear_peer_download(&peer2);
+		assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 80, 1, 200), Some(81 .. 121));
+		bc.clear_peer_download(&peer2);
+		bc.insert(81, blocks[81..121].to_vec(), peer2.clone());
+		bc.clear_peer_download(&peer1);
+		bc.insert(121, blocks[121..150].to_vec(), peer1.clone());
+
+		assert_eq!(bc.drain(80), vec![]);
+		let drained = bc.drain(81);
+		assert_eq!(drained[..40], blocks[81..121].iter()
+			.map(|b| BlockData { block: b.clone(), origin: Some(peer2.clone()) }).collect::<Vec<_>>()[..]);
+		assert_eq!(drained[40..], blocks[121..150].iter()
+			.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
+	}
+
+	#[test]
+	fn large_gap() {
+		let mut bc: BlockCollection<Block> = BlockCollection::new();
+		bc.blocks.insert(100, BlockRangeState::Downloading {
+			len: 128,
+			downloading: 1,
+		});
+		let blocks = generate_blocks(10).into_iter().map(|b| BlockData { block: b, origin: None }).collect();
+		bc.blocks.insert(114305, BlockRangeState::Complete(blocks));
+
+		let peer0 = PeerId::random();
+		assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 000, 1, 200), Some(1 .. 100));
+		assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200), None); // too far ahead
+		assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200000), Some(100 + 128 .. 100 + 128 + 128));
+	}
+}

substrate/client/network/src/protocol/sync/extra_requests.rs

@@ -0,0 +1,511 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use client_api::error::Error as ClientError;
+use crate::protocol::sync::{PeerSync, PeerSyncState};
+use fork_tree::ForkTree;
+use libp2p::PeerId;
+use log::{debug, warn};
+use sr_primitives::traits::{Block as BlockT, NumberFor, Zero};
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::time::{Duration, Instant};
+
+// Time to wait before trying to get the same extra data from the same peer.
+const EXTRA_RETRY_WAIT: Duration = Duration::from_secs(10);
+
+/// Pending extra data request for the given block (hash and number).
+pub(crate) type ExtraRequest<B> = (<B as BlockT>::Hash, NumberFor<B>);
+
+/// Manages pending block extra data (e.g. justification) requests.
+///
+/// Multiple extras may be requested for competing forks, or for the same branch
+/// at different (increasing) heights. This structure will guarantee that extras
+/// are fetched in-order, and that obsolete changes are pruned (when finalizing a
+/// competing fork).
+#[derive(Debug)]
+pub(crate) struct ExtraRequests<B: BlockT> {
+	tree: ForkTree<B::Hash, NumberFor<B>, ()>,
+	/// best finalized block number that we have seen since restart
+	best_seen_finalized_number: NumberFor<B>,
+	/// requests which have been queued for later processing
+	pending_requests: VecDeque<ExtraRequest<B>>,
+	/// requests which are currently underway to some peer
+	active_requests: HashMap<PeerId, ExtraRequest<B>>,
+	/// previous requests without response
+	failed_requests: HashMap<ExtraRequest<B>, Vec<(PeerId, Instant)>>,
+	/// successful requests
+	importing_requests: HashSet<ExtraRequest<B>>,
+}
+
+impl<B: BlockT> ExtraRequests<B> {
+	pub(crate) fn new() -> Self {
+		ExtraRequests {
+			tree: ForkTree::new(),
+			best_seen_finalized_number: Zero::zero(),
+			pending_requests: VecDeque::new(),
+			active_requests: HashMap::new(),
+			failed_requests: HashMap::new(),
+			importing_requests: HashSet::new(),
+		}
+	}
+
+	/// Reset all state as if returned from `new`.
+	pub(crate) fn reset(&mut self) {
+		self.tree = ForkTree::new();
+		self.pending_requests.clear();
+		self.active_requests.clear();
+		self.failed_requests.clear();
+	}
+
+	/// Returns an iterator-like struct that yields peers which extra
+	/// requests can be sent to.
+	pub(crate) fn matcher(&mut self) -> Matcher<B> {
+		Matcher::new(self)
+	}
+
+	/// Queue an extra data request to be considered by the `Matcher`.
+	pub(crate) fn schedule<F>(&mut self, request: ExtraRequest<B>, is_descendent_of: F)
+		where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
+	{
+		match self.tree.import(request.0, request.1, (), &is_descendent_of) {
+			Ok(true) => {
+				// this is a new root so we add it to the current `pending_requests`
+				self.pending_requests.push_back((request.0, request.1));
+			}
+			Err(fork_tree::Error::Revert) => {
+				// we have finalized further than the given request, presumably
+				// by some other part of the system (not sync). we can safely
+				// ignore the `Revert` error.
+				return;
+			},
+			Err(err) => {
+				debug!(target: "sync", "Failed to insert request {:?} into tree: {:?}", request, err);
+				return;
+			}
+			_ => ()
+		}
+	}
+
+	/// Retry any pending request if a peer disconnected.
+	pub(crate) fn peer_disconnected(&mut self, who: &PeerId) {
+		if let Some(request) = self.active_requests.remove(who) {
+			self.pending_requests.push_front(request);
+		}
+	}
+
+	/// Processes the response for the request previously sent to the given peer.
+	pub(crate) fn on_response<R>(&mut self, who: PeerId, resp: Option<R>) -> Option<(PeerId, B::Hash, NumberFor<B>, R)> {
+		// we assume that the request maps to the given response, this is
+		// currently enforced by the outer network protocol before passing on
+		// messages to chain sync.
+		if let Some(request) = self.active_requests.remove(&who) {
+			if let Some(r) = resp {
+				self.importing_requests.insert(request);
+				return Some((who, request.0, request.1, r))
+			}
+			self.failed_requests.entry(request).or_insert(Vec::new()).push((who, Instant::now()));
+			self.pending_requests.push_front(request);
+		}
+		None
+	}
+
+	/// Removes any pending extra requests for blocks lower than the given best finalized.
+	pub(crate) fn on_block_finalized<F>(
+		&mut self,
+		best_finalized_hash: &B::Hash,
+		best_finalized_number: NumberFor<B>,
+		is_descendent_of: F
+	) -> Result<(), fork_tree::Error<ClientError>>
+		where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
+	{
+		let request = (*best_finalized_hash, best_finalized_number);
+
+		if self.try_finalize_root::<()>(request, Ok(request), false) {
+			return Ok(())
+		}
+
+		if best_finalized_number > self.best_seen_finalized_number {
+			// normally we'll receive finality notifications for every block => finalize would be enough
+			// but if many blocks are finalized at once, some notifications may be omitted
+			// => let's use finalize_with_ancestors here
+			match self.tree.finalize_with_ancestors(
+				best_finalized_hash,
+				best_finalized_number,
+				&is_descendent_of,
+			) {
+				Err(fork_tree::Error::Revert) => {
+					// we might have finalized further already in which case we
+					// will get a `Revert` error which we can safely ignore.
+				},
+				Err(err) => return Err(err),
+				Ok(_) => {},
+			}
+
+			self.best_seen_finalized_number = best_finalized_number;
+		}
+
+		let roots = self.tree.roots().collect::<HashSet<_>>();
+
+		self.pending_requests.retain(|(h, n)| roots.contains(&(h, n, &())));
+		self.active_requests.retain(|_, (h, n)| roots.contains(&(h, n, &())));
+		self.failed_requests.retain(|(h, n), _| roots.contains(&(h, n, &())));
+
+		Ok(())
+	}
+
+	/// Try to finalize pending root.
+	///
+	/// Returns true if import of this request has been scheduled.
+	pub(crate) fn try_finalize_root<E>(
+		&mut self,
+		request: ExtraRequest<B>,
+		result: Result<ExtraRequest<B>, E>,
+		reschedule_on_failure: bool
+	) -> bool
+	{
+		if !self.importing_requests.remove(&request) {
+			return false
+		}
+
+		let (finalized_hash, finalized_number) = match result {
+			Ok(req) => (req.0, req.1),
+			Err(_) => {
+				if reschedule_on_failure {
+					self.pending_requests.push_front(request);
+				}
+				return true
+			}
+		};
+
+		if self.tree.finalize_root(&finalized_hash).is_none() {
+			warn!(target: "sync", "Imported {:?} {:?} which isn't a root in the tree: {:?}",
+				finalized_hash,
+				finalized_number,
+				self.tree.roots().collect::<Vec<_>>()
+			);
+			return true
+		}
+
+		self.failed_requests.clear();
+		self.active_requests.clear();
+		self.pending_requests.clear();
+		self.pending_requests.extend(self.tree.roots().map(|(&h, &n, _)| (h, n)));
+		self.best_seen_finalized_number = finalized_number;
+
+		true
+	}
+}
+
+/// Matches peers with pending extra requests.
+#[derive(Debug)]
+pub(crate) struct Matcher<'a, B: BlockT> {
+	/// Length of pending requests collection.
+	/// Used to ensure we do not loop more than once over all pending requests.
+	remaining: usize,
+	extras: &'a mut ExtraRequests<B>
+}
+
+impl<'a, B: BlockT> Matcher<'a, B> {
+	fn new(extras: &'a mut ExtraRequests<B>) -> Self {
+		Matcher {
+			remaining: extras.pending_requests.len(),
+			extras
+		}
+	}
+
+	/// Finds a peer to which a pending request can be sent.
+	///
+	/// Peers are filtered according to the current known best block (i.e. we won't
+	/// send an extra request for block #10 to a peer at block #2), and we also
+	/// throttle requests to the same peer if a previous request yielded no results.
+	///
+	/// This method returns as soon as it finds a peer that should be able to answer
+	/// our request. If no request is pending or no peer can handle it, `None` is
+	/// returned instead.
+	///
+	/// # Note
+	///
+	/// The returned `PeerId` (if any) is guaranteed to come from the given `peers`
+	/// argument.
+	pub(crate) fn next(&mut self, peers: &HashMap<PeerId, PeerSync<B>>) -> Option<(PeerId, ExtraRequest<B>)> {
+		if self.remaining == 0 {
+			return None
+		}
+
+		// clean up previously failed requests so we can retry again
+		for requests in self.extras.failed_requests.values_mut() {
+			requests.retain(|(_, instant)| instant.elapsed() < EXTRA_RETRY_WAIT);
+		}
+
+		while let Some(request) = self.extras.pending_requests.pop_front() {
+			for (peer, sync) in peers.iter().filter(|(_, sync)| sync.state == PeerSyncState::Available) {
+				// only ask peers that have synced at least up to the block number that we're asking the extra for
+				if sync.best_number < request.1 {
+					continue
+				}
+				// don't request to any peers that already have pending requests
+				if self.extras.active_requests.contains_key(peer) {
+					continue
+				}
+				// only ask if the same request has not failed for this peer before
+				if self.extras.failed_requests.get(&request).map(|rr| rr.iter().any(|i| &i.0 == peer)).unwrap_or(false) {
+					continue
+				}
+				self.extras.active_requests.insert(peer.clone(), request);
+				return Some((peer.clone(), request))
+			}
+
+			self.extras.pending_requests.push_back(request);
+			self.remaining -= 1;
+
+			if self.remaining == 0 {
+				break
+			}
+		}
+
+		None
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::protocol::sync::PeerSync;
+	use client_api::error::Error as ClientError;
+	use quickcheck::{Arbitrary, Gen, QuickCheck, StdThreadGen};
+	use rand::Rng;
+	use std::collections::{HashMap, HashSet};
+	use super::*;
+	use test_client::runtime::{Block, BlockNumber, Hash};
+
+	#[test]
+	fn requests_are_processed_in_order() {
+		fn property(mut peers: ArbitraryPeers) {
+			let mut requests = ExtraRequests::<Block>::new();
+
+			let num_peers_available = peers.0.values()
+				.filter(|s| s.state == PeerSyncState::Available).count();
+
+			for i in 0 .. num_peers_available {
+				requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
+			}
+
+			let pending = requests.pending_requests.clone();
+			let mut m = requests.matcher();
+
+			for p in &pending {
+				let (peer, r) = m.next(&peers.0).unwrap();
+				assert_eq!(p, &r);
+				peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
+			}
+		}
+
+		QuickCheck::with_gen(StdThreadGen::new(19))
+			.quickcheck(property as fn(ArbitraryPeers))
+	}
+
+	#[test]
+	fn new_roots_schedule_new_request() {
+		fn property(data: Vec<BlockNumber>) {
+			let mut requests = ExtraRequests::<Block>::new();
+			for (i, number) in data.into_iter().enumerate() {
+				let hash = [i as u8; 32].into();
+				let pending = requests.pending_requests.len();
+				let is_root = requests.tree.roots().any(|(&h, &n, _)| hash == h && number == n);
+				requests.schedule((hash, number), |a, b| Ok(a[0] >= b[0]));
+				if !is_root {
+					assert_eq!(1 + pending, requests.pending_requests.len())
+				}
+			}
+		}
+		QuickCheck::new().quickcheck(property as fn(Vec<BlockNumber>))
+	}
+
+	#[test]
+	fn disconnecting_implies_rescheduling() {
+		fn property(mut peers: ArbitraryPeers) -> bool {
+			let mut requests = ExtraRequests::<Block>::new();
+
+			let num_peers_available = peers.0.values()
+				.filter(|s| s.state == PeerSyncState::Available).count();
+
+			for i in 0 .. num_peers_available {
+				requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
+			}
+
+			let mut m = requests.matcher();
+			while let Some((peer, r)) = m.next(&peers.0) {
+				peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
+			}
+
+			assert!(requests.pending_requests.is_empty());
+
+			let active_peers = requests.active_requests.keys().cloned().collect::<Vec<_>>();
+			let previously_active = requests.active_requests.values().cloned().collect::<HashSet<_>>();
+
+			for peer in &active_peers {
+				requests.peer_disconnected(peer)
+			}
+
+			assert!(requests.active_requests.is_empty());
+
+			previously_active == requests.pending_requests.iter().cloned().collect::<HashSet<_>>()
+		}
+
+		QuickCheck::with_gen(StdThreadGen::new(19))
+			.quickcheck(property as fn(ArbitraryPeers) -> bool)
+	}
+
+	#[test]
+	fn no_response_reschedules() {
+		fn property(mut peers: ArbitraryPeers) {
+			let mut requests = ExtraRequests::<Block>::new();
+
+			let num_peers_available = peers.0.values()
+				.filter(|s| s.state == PeerSyncState::Available).count();
+
+			for i in 0 .. num_peers_available {
+				requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
+			}
+
+			let mut m = requests.matcher();
+			while let Some((peer, r)) = m.next(&peers.0) {
+				peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
+			}
+
+			let active = requests.active_requests.iter().map(|(p, &r)| (p.clone(), r)).collect::<Vec<_>>();
+
+			for (peer, req) in &active {
+				assert!(requests.failed_requests.get(req).is_none());
+				assert!(!requests.pending_requests.contains(req));
+				assert!(requests.on_response::<()>(peer.clone(), None).is_none());
+				assert!(requests.pending_requests.contains(req));
+				assert_eq!(1, requests.failed_requests.get(req).unwrap().iter().filter(|(p, _)| p == peer).count())
+			}
+		}
+
+		QuickCheck::with_gen(StdThreadGen::new(19))
+			.quickcheck(property as fn(ArbitraryPeers))
+	}
+
+	#[test]
+	fn request_is_rescheduled_when_earlier_block_is_finalized() {
+		let _ = ::env_logger::try_init();
+
+		let mut finality_proofs = ExtraRequests::<Block>::new();
+
+		let hash4 = [4; 32].into();
+		let hash5 = [5; 32].into();
+		let hash6 = [6; 32].into();
+		let hash7 = [7; 32].into();
+
+		fn is_descendent_of(base: &Hash, target: &Hash) -> Result<bool, ClientError> {
+			Ok(target[0] >= base[0])
+		}
+
+		// make #4 last finalized block
+		finality_proofs.tree.import(hash4, 4, (), &is_descendent_of).unwrap();
+		finality_proofs.tree.finalize_root(&hash4);
+
+		// schedule request for #6
+		finality_proofs.schedule((hash6, 6), is_descendent_of);
+
+		// receive finality proof for #5
+		finality_proofs.importing_requests.insert((hash6, 6));
+		finality_proofs.on_block_finalized(&hash5, 5, is_descendent_of).unwrap();
+		finality_proofs.try_finalize_root::<()>((hash6, 6), Ok((hash5, 5)), true);
+
+		// ensure that request for #6 is still pending
+		assert_eq!(finality_proofs.pending_requests.iter().collect::<Vec<_>>(), vec![&(hash6, 6)]);
+
+		// receive finality proof for #7
+		finality_proofs.importing_requests.insert((hash6, 6));
+		finality_proofs.on_block_finalized(&hash6, 6, is_descendent_of).unwrap();
+		finality_proofs.on_block_finalized(&hash7, 7, is_descendent_of).unwrap();
+		finality_proofs.try_finalize_root::<()>((hash6, 6), Ok((hash7, 7)), true);
+
+		// ensure that there's no request for #6
+		assert_eq!(finality_proofs.pending_requests.iter().collect::<Vec<_>>(), Vec::<&(Hash, u64)>::new());
+	}
+
+	#[test]
+	fn anecstor_roots_are_finalized_when_finality_notification_is_missed() {
+		let mut finality_proofs = ExtraRequests::<Block>::new();
+
+		let hash4 = [4; 32].into();
+		let hash5 = [5; 32].into();
+
+		fn is_descendent_of(base: &Hash, target: &Hash) -> Result<bool, ClientError> {
+			Ok(target[0] >= base[0])
+		}
+
+		// schedule request for #4
+		finality_proofs.schedule((hash4, 4), is_descendent_of);
+
+		// receive finality notification for #5 (missing notification for #4!!!)
+		finality_proofs.importing_requests.insert((hash4, 5));
+		finality_proofs.on_block_finalized(&hash5, 5, is_descendent_of).unwrap();
+		assert_eq!(finality_proofs.tree.roots().count(), 0);
+	}
+
+	// Some Arbitrary instances to allow easy construction of random peer sets:
+
+	#[derive(Debug, Clone)]
+	struct ArbitraryPeerSyncState(PeerSyncState<Block>);
+
+	impl Arbitrary for ArbitraryPeerSyncState {
+		fn arbitrary<G: Gen>(g: &mut G) -> Self {
+			let s = match g.gen::<u8>() % 5 {
+				0 => PeerSyncState::Available,
+				// TODO: 1 => PeerSyncState::AncestorSearch(g.gen(), AncestorSearchState<B>),
+				1 => PeerSyncState::DownloadingNew(g.gen::<BlockNumber>()),
+				2 => PeerSyncState::DownloadingStale(Hash::random()),
+				3 => PeerSyncState::DownloadingJustification(Hash::random()),
+				_ => PeerSyncState::DownloadingFinalityProof(Hash::random())
+			};
+			ArbitraryPeerSyncState(s)
+		}
+	}
+
+	#[derive(Debug, Clone)]
+	struct ArbitraryPeerSync(PeerSync<Block>);
+
+	impl Arbitrary for ArbitraryPeerSync {
+		fn arbitrary<G: Gen>(g: &mut G) -> Self {
+			let ps = PeerSync {
+				common_number: g.gen(),
+				best_hash: Hash::random(),
+				best_number: g.gen(),
+				state: ArbitraryPeerSyncState::arbitrary(g).0,
+				recently_announced: Default::default()
+			};
+			ArbitraryPeerSync(ps)
+		}
+	}
+
+	#[derive(Debug, Clone)]
+	struct ArbitraryPeers(HashMap<PeerId, PeerSync<Block>>);
+
+	impl Arbitrary for ArbitraryPeers {
+		fn arbitrary<G: Gen>(g: &mut G) -> Self {
+			let mut peers = HashMap::with_capacity(g.size());
+			for _ in 0 .. g.size() {
+				peers.insert(PeerId::random(), ArbitraryPeerSync::arbitrary(g).0);
+			}
+			ArbitraryPeers(peers)
+		}
+	}
+
+}

substrate/client/network/src/protocol/util.rs

@@ -0,0 +1,76 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use linked_hash_set::LinkedHashSet;
+use std::{hash::Hash, num::NonZeroUsize};
+
+/// Wrapper around `LinkedHashSet` which grows bounded.
+///
+/// In the limit, for each element inserted the oldest existing element will be removed.
+#[derive(Debug, Clone)]
+pub(crate) struct LruHashSet<T: Hash + Eq> {
+	set: LinkedHashSet<T>,
+	limit: NonZeroUsize
+}
+
+impl<T: Hash + Eq> LruHashSet<T> {
+	/// Create a new `LruHashSet` with the given (exclusive) limit.
+	pub(crate) fn new(limit: NonZeroUsize) -> Self {
+		Self { set: LinkedHashSet::new(), limit }
+	}
+
+	/// Insert element into the set.
+	///
+	/// Returns `true` if this is a new element to the set, `false` otherwise.
+	/// Maintains the limit of the set by removing the oldest entry if necessary.
+	/// Inserting the same element will update its LRU position.
+	pub(crate) fn insert(&mut self, e: T) -> bool {
+		if self.set.insert(e) {
+			if self.set.len() == usize::from(self.limit) {
+				self.set.pop_front(); // remove oldest entry
+			}
+			return true
+		}
+		false
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn maintains_limit() {
+		let three = NonZeroUsize::new(3).unwrap();
+		let mut set = LruHashSet::<u8>::new(three);
+
+		// First element.
+		assert!(set.insert(1));
+		assert_eq!(vec![&1], set.set.iter().collect::<Vec<_>>());
+
+		// Second element.
+		assert!(set.insert(2));
+		assert_eq!(vec![&1, &2], set.set.iter().collect::<Vec<_>>());
+
+		// Inserting the same element updates its LRU position.
+		assert!(!set.insert(1));
+		assert_eq!(vec![&2, &1], set.set.iter().collect::<Vec<_>>());
+
+		// We reached the limit. The next element forces the oldest one out.
+		assert!(set.insert(3));
+		assert_eq!(vec![&1, &3], set.set.iter().collect::<Vec<_>>());
+	}
+}

substrate/client/network/src/service.rs

@@ -0,0 +1,812 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Main entry point of the substrate-network crate.
+//!
+//! There are two main structs in this module: [`NetworkWorker`] and [`NetworkService`].
+//! The [`NetworkWorker`] *is* the network and implements the `Future` trait. It must be polled in
+//! order fo the network to advance.
+//! The [`NetworkService`] is merely a shared version of the [`NetworkWorker`]. You can obtain an
+//! `Arc<NetworkService>` by calling [`NetworkWorker::service`].
+//!
+//! The methods of the [`NetworkService`] are implemented by sending a message over a channel,
+//! which is then processed by [`NetworkWorker::poll`].
+
+use std::{collections::{HashMap, HashSet}, fs, marker::PhantomData, io, path::Path};
+use std::sync::{Arc, atomic::{AtomicBool, AtomicUsize, Ordering}};
+
+use consensus::import_queue::{ImportQueue, Link};
+use consensus::import_queue::{BlockImportResult, BlockImportError};
+use futures::{prelude::*, sync::mpsc};
+use futures03::TryFutureExt as _;
+use log::{warn, error, info};
+use libp2p::{PeerId, Multiaddr, kad::record};
+use libp2p::core::{transport::boxed::Boxed, muxing::StreamMuxerBox};
+use libp2p::swarm::NetworkBehaviour;
+use parking_lot::Mutex;
+use peerset::PeersetHandle;
+use sr_primitives::{traits::{Block as BlockT, NumberFor}, ConsensusEngineId};
+
+use crate::{behaviour::{Behaviour, BehaviourOut}, config::{parse_str_addr, parse_addr}};
+use crate::{NetworkState, NetworkStateNotConnectedPeer, NetworkStatePeer};
+use crate::{transport, config::NonReservedPeerMode};
+use crate::config::{Params, TransportConfig};
+use crate::error::Error;
+use crate::protocol::{self, Protocol, Context, CustomMessageOutcome, PeerInfo};
+use crate::protocol::consensus_gossip::{ConsensusGossip, MessageRecipient as GossipMessageRecipient};
+use crate::protocol::{event::Event, light_dispatch::{AlwaysBadChecker, RequestData}};
+use crate::protocol::specialization::NetworkSpecialization;
+use crate::protocol::sync::SyncState;
+
+/// Minimum Requirements for a Hash within Networking
+pub trait ExHashT: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static {}
+
+impl<T> ExHashT for T where
+	T: std::hash::Hash + Eq + std::fmt::Debug + Clone + Send + Sync + 'static
+{}
+
+/// Transaction pool interface
+pub trait TransactionPool<H: ExHashT, B: BlockT>: Send + Sync {
+	/// Get transactions from the pool that are ready to be propagated.
+	fn transactions(&self) -> Vec<(H, B::Extrinsic)>;
+	/// Get hash of transaction.
+	fn hash_of(&self, transaction: &B::Extrinsic) -> H;
+	/// Import a transaction into the pool.
+	///
+	/// Peer reputation is changed by reputation_change if transaction is accepted by the pool.
+	fn import(
+		&self,
+		report_handle: ReportHandle,
+		who: PeerId,
+		reputation_change_good: i32,
+		reputation_change_bad: i32,
+		transaction: B::Extrinsic,
+	);
+	/// Notify the pool about transactions broadcast.
+	fn on_broadcasted(&self, propagations: HashMap<H, Vec<String>>);
+}
+
+/// A cloneable handle for reporting cost/benefits of peers.
+#[derive(Clone)]
+pub struct ReportHandle {
+	inner: PeersetHandle, // wraps it so we don't have to worry about breaking API.
+}
+
+impl From<PeersetHandle> for ReportHandle {
+	fn from(peerset_handle: PeersetHandle) -> Self {
+		ReportHandle { inner: peerset_handle }
+	}
+}
+
+impl ReportHandle {
+	/// Report a given peer as either beneficial (+) or costly (-) according to the
+	/// given scalar.
+	pub fn report_peer(&self, who: PeerId, cost_benefit: i32) {
+		self.inner.report_peer(who, cost_benefit);
+	}
+}
+
+/// Substrate network service. Handles network IO and manages connectivity.
+pub struct NetworkService<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> {
+	/// Number of peers we're connected to.
+	num_connected: Arc<AtomicUsize>,
+	/// The local external addresses.
+	external_addresses: Arc<Mutex<Vec<Multiaddr>>>,
+	/// Are we actively catching up with the chain?
+	is_major_syncing: Arc<AtomicBool>,
+	/// Local copy of the `PeerId` of the local node.
+	local_peer_id: PeerId,
+	/// Bandwidth logging system. Can be queried to know the average bandwidth consumed.
+	bandwidth: Arc<transport::BandwidthSinks>,
+	/// Peerset manager (PSM); manages the reputation of nodes and indicates the network which
+	/// nodes it should be connected to or not.
+	peerset: PeersetHandle,
+	/// Channel that sends messages to the actual worker.
+	to_worker: mpsc::UnboundedSender<ServerToWorkerMsg<B, S>>,
+	/// Marker to pin the `H` generic. Serves no purpose except to not break backwards
+	/// compatibility.
+	_marker: PhantomData<H>,
+}
+
+impl<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> NetworkWorker<B, S, H> {
+	/// Creates the network service.
+	///
+	/// Returns a `NetworkWorker` that implements `Future` and must be regularly polled in order
+	/// for the network processing to advance. From it, you can extract a `NetworkService` using
+	/// `worker.service()`. The `NetworkService` can be shared through the codebase.
+	pub fn new(params: Params<B, S, H>) -> Result<NetworkWorker<B, S, H>, Error> {
+		let (to_worker, from_worker) = mpsc::unbounded();
+
+		if let Some(ref path) = params.network_config.net_config_path {
+			fs::create_dir_all(Path::new(path))?;
+		}
+
+		// List of multiaddresses that we know in the network.
+		let mut known_addresses = Vec::new();
+		let mut bootnodes = Vec::new();
+		let mut reserved_nodes = Vec::new();
+
+		// Process the bootnodes.
+		for bootnode in params.network_config.boot_nodes.iter() {
+			match parse_str_addr(bootnode) {
+				Ok((peer_id, addr)) => {
+					bootnodes.push(peer_id.clone());
+					known_addresses.push((peer_id, addr));
+				},
+				Err(_) => warn!(target: "sub-libp2p", "Not a valid bootnode address: {}", bootnode),
+			}
+		}
+
+		// Check for duplicate bootnodes.
+		known_addresses.iter()
+			.try_for_each(|(peer_id, addr)|
+				if let Some(other) = known_addresses
+					.iter()
+					.find(|o| o.1 == *addr && o.0 != *peer_id)
+				{
+					Err(Error::DuplicateBootnode {
+						address: addr.clone(),
+						first_id: peer_id.clone(),
+						second_id: other.0.clone(),
+					})
+				} else {
+					Ok(())
+				}
+			)?;
+
+		// Initialize the reserved peers.
+		for reserved in params.network_config.reserved_nodes.iter() {
+			if let Ok((peer_id, addr)) = parse_str_addr(reserved) {
+				reserved_nodes.push(peer_id.clone());
+				known_addresses.push((peer_id, addr));
+			} else {
+				warn!(target: "sub-libp2p", "Not a valid reserved node address: {}", reserved);
+			}
+		}
+
+		let peerset_config = peerset::PeersetConfig {
+			in_peers: params.network_config.in_peers,
+			out_peers: params.network_config.out_peers,
+			bootnodes,
+			reserved_only: params.network_config.non_reserved_mode == NonReservedPeerMode::Deny,
+			reserved_nodes,
+		};
+
+		// Private and public keys configuration.
+		let local_identity = params.network_config.node_key.clone().into_keypair()?;
+		let local_public = local_identity.public();
+		let local_peer_id = local_public.clone().into_peer_id();
+		info!(target: "sub-libp2p", "Local node identity is: {}", local_peer_id.to_base58());
+
+		let num_connected = Arc::new(AtomicUsize::new(0));
+		let is_major_syncing = Arc::new(AtomicBool::new(false));
+		let (protocol, peerset_handle) = Protocol::new(
+			protocol::ProtocolConfig {
+				roles: params.roles,
+				max_parallel_downloads: params.network_config.max_parallel_downloads,
+			},
+			params.chain,
+			params.on_demand.as_ref().map(|od| od.checker().clone())
+				.unwrap_or(Arc::new(AlwaysBadChecker)),
+			params.specialization,
+			params.transaction_pool,
+			params.finality_proof_provider,
+			params.finality_proof_request_builder,
+			params.protocol_id,
+			peerset_config,
+			params.block_announce_validator
+		)?;
+
+		// Build the swarm.
+		let (mut swarm, bandwidth) = {
+			let user_agent = format!(
+				"{} ({})",
+				params.network_config.client_version,
+				params.network_config.node_name
+			);
+			let behaviour = Behaviour::new(
+				protocol,
+				user_agent,
+				local_public,
+				known_addresses,
+				match params.network_config.transport {
+					TransportConfig::MemoryOnly => false,
+					TransportConfig::Normal { enable_mdns, .. } => enable_mdns,
+				},
+				match params.network_config.transport {
+					TransportConfig::MemoryOnly => false,
+					TransportConfig::Normal { allow_private_ipv4, .. } => allow_private_ipv4,
+				},
+			);
+			let (transport, bandwidth) = {
+				let (config_mem, config_wasm) = match params.network_config.transport {
+					TransportConfig::MemoryOnly => (true, None),
+					TransportConfig::Normal { wasm_external_transport, .. } =>
+						(false, wasm_external_transport)
+				};
+				transport::build_transport(local_identity, config_mem, config_wasm)
+			};
+			(Swarm::<B, S, H>::new(transport, behaviour, local_peer_id.clone()), bandwidth)
+		};
+
+		// Listen on multiaddresses.
+		for addr in &params.network_config.listen_addresses {
+			if let Err(err) = Swarm::<B, S, H>::listen_on(&mut swarm, addr.clone()) {
+				warn!(target: "sub-libp2p", "Can't listen on {} because: {:?}", addr, err)
+			}
+		}
+
+		// Add external addresses.
+		for addr in &params.network_config.public_addresses {
+			Swarm::<B, S, H>::add_external_address(&mut swarm, addr.clone());
+		}
+
+		let external_addresses = Arc::new(Mutex::new(Vec::new()));
+
+		let service = Arc::new(NetworkService {
+			bandwidth,
+			external_addresses: external_addresses.clone(),
+			num_connected: num_connected.clone(),
+			is_major_syncing: is_major_syncing.clone(),
+			peerset: peerset_handle,
+			local_peer_id,
+			to_worker: to_worker.clone(),
+			_marker: PhantomData,
+		});
+
+		Ok(NetworkWorker {
+			external_addresses,
+			num_connected,
+			is_major_syncing,
+			network_service: swarm,
+			service,
+			import_queue: params.import_queue,
+			from_worker,
+			light_client_rqs: params.on_demand.and_then(|od| od.extract_receiver()),
+		})
+	}
+
+	/// Returns the downloaded bytes per second averaged over the past few seconds.
+	pub fn average_download_per_sec(&self) -> u64 {
+		self.service.bandwidth.average_download_per_sec()
+	}
+
+	/// Returns the uploaded bytes per second averaged over the past few seconds.
+	pub fn average_upload_per_sec(&self) -> u64 {
+		self.service.bandwidth.average_upload_per_sec()
+	}
+
+	/// Returns the number of peers we're connected to.
+	pub fn num_connected_peers(&self) -> usize {
+		self.network_service.user_protocol().num_connected_peers()
+	}
+
+	/// Returns the number of peers we're connected to and that are being queried.
+	pub fn num_active_peers(&self) -> usize {
+		self.network_service.user_protocol().num_active_peers()
+	}
+
+	/// Current global sync state.
+	pub fn sync_state(&self) -> SyncState {
+		self.network_service.user_protocol().sync_state()
+	}
+
+	/// Target sync block number.
+	pub fn best_seen_block(&self) -> Option<NumberFor<B>> {
+		self.network_service.user_protocol().best_seen_block()
+	}
+
+	/// Number of peers participating in syncing.
+	pub fn num_sync_peers(&self) -> u32 {
+		self.network_service.user_protocol().num_sync_peers()
+	}
+
+	/// Number of blocks in the import queue.
+	pub fn num_queued_blocks(&self) -> u32 {
+		self.network_service.user_protocol().num_queued_blocks()
+	}
+
+	/// Adds an address for a node.
+	pub fn add_known_address(&mut self, peer_id: PeerId, addr: Multiaddr) {
+		self.network_service.add_known_address(peer_id, addr);
+	}
+
+	/// Return a `NetworkService` that can be shared through the code base and can be used to
+	/// manipulate the worker.
+	pub fn service(&self) -> &Arc<NetworkService<B, S, H>> {
+		&self.service
+	}
+
+	/// You must call this when a new block is imported by the client.
+	pub fn on_block_imported(&mut self, hash: B::Hash, header: B::Header, data: Vec<u8>, is_best: bool) {
+		self.network_service.user_protocol_mut().on_block_imported(hash, &header, data, is_best);
+	}
+
+	/// You must call this when a new block is finalized by the client.
+	pub fn on_block_finalized(&mut self, hash: B::Hash, header: B::Header) {
+		self.network_service.user_protocol_mut().on_block_finalized(hash, &header);
+	}
+
+	/// Get network state.
+	///
+	/// **Note**: Use this only for debugging. This API is unstable. There are warnings literaly
+	/// everywhere about this. Please don't use this function to retrieve actual information.
+	pub fn network_state(&mut self) -> NetworkState {
+		let swarm = &mut self.network_service;
+		let open = swarm.user_protocol().open_peers().cloned().collect::<Vec<_>>();
+
+		let connected_peers = {
+			let swarm = &mut *swarm;
+			open.iter().filter_map(move |peer_id| {
+				let known_addresses = NetworkBehaviour::addresses_of_peer(&mut **swarm, peer_id)
+					.into_iter().collect();
+
+				let endpoint = if let Some(e) = swarm.node(peer_id).map(|i| i.endpoint()) {
+					e.clone().into()
+				} else {
+					error!(target: "sub-libp2p", "Found state inconsistency between custom protocol \
+						and debug information about {:?}", peer_id);
+					return None
+				};
+
+				Some((peer_id.to_base58(), NetworkStatePeer {
+					endpoint,
+					version_string: swarm.node(peer_id)
+						.and_then(|i| i.client_version().map(|s| s.to_owned())).clone(),
+					latest_ping_time: swarm.node(peer_id).and_then(|i| i.latest_ping()),
+					enabled: swarm.user_protocol().is_enabled(&peer_id),
+					open: swarm.user_protocol().is_open(&peer_id),
+					known_addresses,
+				}))
+			}).collect()
+		};
+
+		let not_connected_peers = {
+			let swarm = &mut *swarm;
+			let list = swarm.known_peers().filter(|p| open.iter().all(|n| n != *p))
+				.cloned().collect::<Vec<_>>();
+			list.into_iter().map(move |peer_id| {
+				(peer_id.to_base58(), NetworkStateNotConnectedPeer {
+					version_string: swarm.node(&peer_id)
+						.and_then(|i| i.client_version().map(|s| s.to_owned())).clone(),
+					latest_ping_time: swarm.node(&peer_id).and_then(|i| i.latest_ping()),
+					known_addresses: NetworkBehaviour::addresses_of_peer(&mut **swarm, &peer_id)
+						.into_iter().collect(),
+				})
+			}).collect()
+		};
+
+		NetworkState {
+			peer_id: Swarm::<B, S, H>::local_peer_id(&swarm).to_base58(),
+			listened_addresses: Swarm::<B, S, H>::listeners(&swarm).cloned().collect(),
+			external_addresses: Swarm::<B, S, H>::external_addresses(&swarm).cloned().collect(),
+			average_download_per_sec: self.service.bandwidth.average_download_per_sec(),
+			average_upload_per_sec: self.service.bandwidth.average_upload_per_sec(),
+			connected_peers,
+			not_connected_peers,
+			peerset: swarm.user_protocol_mut().peerset_debug_info(),
+		}
+	}
+
+	/// Get currently connected peers.
+	pub fn peers_debug_info(&mut self) -> Vec<(PeerId, PeerInfo<B>)> {
+		self.network_service.user_protocol_mut()
+			.peers_info()
+			.map(|(id, info)| (id.clone(), info.clone()))
+			.collect()
+	}
+}
+
+impl<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> NetworkService<B, S, H> {
+	/// Returns the network identity of the node.
+	pub fn local_peer_id(&self) -> PeerId {
+		self.local_peer_id.clone()
+	}
+
+	/// You must call this when new transactons are imported by the transaction pool.
+	///
+	/// The latest transactions will be fetched from the `TransactionPool` that was passed at
+	/// initialization as part of the configuration.
+	pub fn trigger_repropagate(&self) {
+		let _ = self.to_worker.unbounded_send(ServerToWorkerMsg::PropagateExtrinsics);
+	}
+
+	/// Make sure an important block is propagated to peers.
+	///
+	/// In chain-based consensus, we often need to make sure non-best forks are
+	/// at least temporarily synced. This function forces such an announcement.
+	pub fn announce_block(&self, hash: B::Hash, data: Vec<u8>) {
+		let _ = self.to_worker.unbounded_send(ServerToWorkerMsg::AnnounceBlock(hash, data));
+	}
+
+	/// Send a consensus message through the gossip
+	pub fn gossip_consensus_message(
+		&self,
+		topic: B::Hash,
+		engine_id: ConsensusEngineId,
+		message: Vec<u8>,
+		recipient: GossipMessageRecipient,
+	) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::GossipConsensusMessage(
+				topic, engine_id, message, recipient,
+			));
+	}
+
+	/// Report a given peer as either beneficial (+) or costly (-) according to the
+	/// given scalar.
+	pub fn report_peer(&self, who: PeerId, cost_benefit: i32) {
+		self.peerset.report_peer(who, cost_benefit);
+	}
+
+	/// Request a justification for the given block from the network.
+	///
+	/// On success, the justification will be passed to the import queue that was part at
+	/// initialization as part of the configuration.
+	pub fn request_justification(&self, hash: &B::Hash, number: NumberFor<B>) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::RequestJustification(hash.clone(), number));
+	}
+
+	/// Execute a closure with the chain-specific network specialization.
+	pub fn with_spec<F>(&self, f: F)
+		where F: FnOnce(&mut S, &mut dyn Context<B>) + Send + 'static
+	{
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::ExecuteWithSpec(Box::new(f)));
+	}
+
+	/// Execute a closure with the consensus gossip.
+	pub fn with_gossip<F>(&self, f: F)
+		where F: FnOnce(&mut ConsensusGossip<B>, &mut dyn Context<B>) + Send + 'static
+	{
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::ExecuteWithGossip(Box::new(f)));
+	}
+
+	/// Are we in the process of downloading the chain?
+	pub fn is_major_syncing(&self) -> bool {
+		self.is_major_syncing.load(Ordering::Relaxed)
+	}
+
+	/// Start getting a value from the DHT.
+	///
+	/// This will generate either a `ValueFound` or a `ValueNotFound` event and pass it as an
+	/// item on the [`NetworkWorker`] stream.
+	pub fn get_value(&self, key: &record::Key) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::GetValue(key.clone()));
+	}
+
+	/// Start putting a value in the DHT.
+	///
+	/// This will generate either a `ValuePut` or a `ValuePutFailed` event and pass it as an
+	/// item on the [`NetworkWorker`] stream.
+	pub fn put_value(&self, key: record::Key, value: Vec<u8>) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::PutValue(key, value));
+	}
+
+	/// Connect to unreserved peers and allow unreserved peers to connect.
+	pub fn accept_unreserved_peers(&self) {
+		self.peerset.set_reserved_only(false);
+	}
+
+	/// Disconnect from unreserved peers and deny new unreserved peers to connect.
+	pub fn deny_unreserved_peers(&self) {
+		self.peerset.set_reserved_only(true);
+	}
+
+	/// Removes a `PeerId` from the list of reserved peers.
+	pub fn remove_reserved_peer(&self, peer: PeerId) {
+		self.peerset.remove_reserved_peer(peer);
+	}
+
+	/// Adds a `PeerId` and its address as reserved.
+	pub fn add_reserved_peer(&self, peer: String) -> Result<(), String> {
+		let (peer_id, addr) = parse_str_addr(&peer).map_err(|e| format!("{:?}", e))?;
+		self.peerset.add_reserved_peer(peer_id.clone());
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::AddKnownAddress(peer_id, addr));
+		Ok(())
+	}
+
+	/// Configure an explicit fork sync request.
+	/// Note that this function should not be used for recent blocks.
+	/// Sync should be able to download all the recent forks normally.
+	/// `set_sync_fork_request` should only be used if external code detects that there's
+	/// a stale fork missing.
+	/// Passing empty `peers` set effectively removes the sync request.
+	pub fn set_sync_fork_request(&self, peers: Vec<PeerId>, hash: B::Hash, number: NumberFor<B>) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServerToWorkerMsg::SyncFork(peers, hash, number));
+	}
+
+	/// Modify a peerset priority group.
+	pub fn set_priority_group(&self, group_id: String, peers: HashSet<Multiaddr>) -> Result<(), String> {
+		let peers = peers.into_iter().map(|p| {
+			parse_addr(p).map_err(|e| format!("{:?}", e))
+		}).collect::<Result<Vec<(PeerId, Multiaddr)>, String>>()?;
+
+		let peer_ids = peers.iter().map(|(peer_id, _addr)| peer_id.clone()).collect();
+		self.peerset.set_priority_group(group_id, peer_ids);
+
+		for (peer_id, addr) in peers.into_iter() {
+			let _ = self
+				.to_worker
+				.unbounded_send(ServerToWorkerMsg::AddKnownAddress(peer_id, addr));
+		}
+
+		Ok(())
+	}
+
+	/// Returns the number of peers we're connected to.
+	pub fn num_connected(&self) -> usize {
+		self.num_connected.load(Ordering::Relaxed)
+	}
+
+	/// Returns the local external addresses.
+	pub fn external_addresses(&self) -> Vec<Multiaddr> {
+		self.external_addresses.lock().clone()
+	}
+}
+
+impl<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> consensus::SyncOracle
+	for NetworkService<B, S, H>
+{
+	fn is_major_syncing(&mut self) -> bool {
+		NetworkService::is_major_syncing(self)
+	}
+
+	fn is_offline(&mut self) -> bool {
+		self.num_connected.load(Ordering::Relaxed) == 0
+	}
+}
+
+impl<'a, B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> consensus::SyncOracle
+	for &'a NetworkService<B, S, H>
+{
+	fn is_major_syncing(&mut self) -> bool {
+		NetworkService::is_major_syncing(self)
+	}
+
+	fn is_offline(&mut self) -> bool {
+		self.num_connected.load(Ordering::Relaxed) == 0
+	}
+}
+
+/// Trait for providing information about the local network state
+pub trait NetworkStateInfo {
+	/// Returns the local external addresses.
+	fn external_addresses(&self) -> Vec<Multiaddr>;
+
+	/// Returns the local Peer ID.
+	fn peer_id(&self) -> PeerId;
+}
+
+impl<B, S, H> NetworkStateInfo for NetworkService<B, S, H>
+	where
+		B: sr_primitives::traits::Block,
+		S: NetworkSpecialization<B>,
+		H: ExHashT,
+{
+	/// Returns the local external addresses.
+	fn external_addresses(&self) -> Vec<Multiaddr> {
+		self.external_addresses.lock().clone()
+	}
+
+	/// Returns the local Peer ID.
+	fn peer_id(&self) -> PeerId {
+		self.local_peer_id.clone()
+	}
+}
+
+/// Messages sent from the `NetworkService` to the `NetworkWorker`.
+///
+/// Each entry corresponds to a method of `NetworkService`.
+enum ServerToWorkerMsg<B: BlockT, S: NetworkSpecialization<B>> {
+	PropagateExtrinsics,
+	RequestJustification(B::Hash, NumberFor<B>),
+	AnnounceBlock(B::Hash, Vec<u8>),
+	ExecuteWithSpec(Box<dyn FnOnce(&mut S, &mut dyn Context<B>) + Send>),
+	ExecuteWithGossip(Box<dyn FnOnce(&mut ConsensusGossip<B>, &mut dyn Context<B>) + Send>),
+	GossipConsensusMessage(B::Hash, ConsensusEngineId, Vec<u8>, GossipMessageRecipient),
+	GetValue(record::Key),
+	PutValue(record::Key, Vec<u8>),
+	AddKnownAddress(PeerId, Multiaddr),
+	SyncFork(Vec<PeerId>, B::Hash, NumberFor<B>),
+}
+
+/// Main network worker. Must be polled in order for the network to advance.
+///
+/// You are encouraged to poll this in a separate background thread or task.
+#[must_use = "The NetworkWorker must be polled in order for the network to work"]
+pub struct NetworkWorker<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> {
+	/// Updated by the `NetworkWorker` and loaded by the `NetworkService`.
+	external_addresses: Arc<Mutex<Vec<Multiaddr>>>,
+	/// Updated by the `NetworkWorker` and loaded by the `NetworkService`.
+	num_connected: Arc<AtomicUsize>,
+	/// Updated by the `NetworkWorker` and loaded by the `NetworkService`.
+	is_major_syncing: Arc<AtomicBool>,
+	/// The network service that can be extracted and shared through the codebase.
+	service: Arc<NetworkService<B, S, H>>,
+	/// The *actual* network.
+	network_service: Swarm<B, S, H>,
+	/// The import queue that was passed as initialization.
+	import_queue: Box<dyn ImportQueue<B>>,
+	/// Messages from the `NetworkService` and that must be processed.
+	from_worker: mpsc::UnboundedReceiver<ServerToWorkerMsg<B, S>>,
+	/// Receiver for queries from the light client that must be processed.
+	light_client_rqs: Option<mpsc::UnboundedReceiver<RequestData<B>>>,
+}
+
+impl<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> Stream for NetworkWorker<B, S, H> {
+	type Item = Event;
+	type Error = io::Error;
+
+	fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
+		// Poll the import queue for actions to perform.
+		let _ = futures03::future::poll_fn(|cx| {
+			self.import_queue.poll_actions(cx, &mut NetworkLink {
+				protocol: &mut self.network_service,
+			});
+			std::task::Poll::Pending::<Result<(), ()>>
+		}).compat().poll();
+
+		// Check for new incoming light client requests.
+		if let Some(light_client_rqs) = self.light_client_rqs.as_mut() {
+			while let Ok(Async::Ready(Some(rq))) = light_client_rqs.poll() {
+				self.network_service.user_protocol_mut().add_light_client_request(rq);
+			}
+		}
+
+		loop {
+			// Process the next message coming from the `NetworkService`.
+			let msg = match self.from_worker.poll() {
+				Ok(Async::Ready(Some(msg))) => msg,
+				Ok(Async::Ready(None)) | Err(_) => return Ok(Async::Ready(None)),
+				Ok(Async::NotReady) => break,
+			};
+
+			match msg {
+				ServerToWorkerMsg::ExecuteWithSpec(task) => {
+					let protocol = self.network_service.user_protocol_mut();
+					let (mut context, spec) = protocol.specialization_lock();
+					task(spec, &mut context);
+				},
+				ServerToWorkerMsg::ExecuteWithGossip(task) => {
+					let protocol = self.network_service.user_protocol_mut();
+					let (mut context, gossip) = protocol.consensus_gossip_lock();
+					task(gossip, &mut context);
+				}
+				ServerToWorkerMsg::GossipConsensusMessage(topic, engine_id, message, recipient) =>
+					self.network_service.user_protocol_mut().gossip_consensus_message(topic, engine_id, message, recipient),
+				ServerToWorkerMsg::AnnounceBlock(hash, data) =>
+					self.network_service.user_protocol_mut().announce_block(hash, data),
+				ServerToWorkerMsg::RequestJustification(hash, number) =>
+					self.network_service.user_protocol_mut().request_justification(&hash, number),
+				ServerToWorkerMsg::PropagateExtrinsics =>
+					self.network_service.user_protocol_mut().propagate_extrinsics(),
+				ServerToWorkerMsg::GetValue(key) =>
+					self.network_service.get_value(&key),
+				ServerToWorkerMsg::PutValue(key, value) =>
+					self.network_service.put_value(key, value),
+				ServerToWorkerMsg::AddKnownAddress(peer_id, addr) =>
+					self.network_service.add_known_address(peer_id, addr),
+				ServerToWorkerMsg::SyncFork(peer_ids, hash, number) =>
+					self.network_service.user_protocol_mut().set_sync_fork_request(peer_ids, &hash, number),
+			}
+		}
+
+		loop {
+			// Process the next action coming from the network.
+			let poll_value = self.network_service.poll();
+
+			let outcome = match poll_value {
+				Ok(Async::NotReady) => break,
+				Ok(Async::Ready(Some(BehaviourOut::SubstrateAction(outcome)))) => outcome,
+				Ok(Async::Ready(Some(BehaviourOut::Dht(ev)))) =>
+					return Ok(Async::Ready(Some(Event::Dht(ev)))),
+				Ok(Async::Ready(None)) => CustomMessageOutcome::None,
+				Err(err) => {
+					error!(target: "sync", "Error in the network: {:?}", err);
+					return Err(err)
+				}
+			};
+
+			match outcome {
+				CustomMessageOutcome::BlockImport(origin, blocks) =>
+					self.import_queue.import_blocks(origin, blocks),
+				CustomMessageOutcome::JustificationImport(origin, hash, nb, justification) =>
+					self.import_queue.import_justification(origin, hash, nb, justification),
+				CustomMessageOutcome::FinalityProofImport(origin, hash, nb, proof) =>
+					self.import_queue.import_finality_proof(origin, hash, nb, proof),
+				CustomMessageOutcome::None => {}
+			}
+		}
+
+		// Update the variables shared with the `NetworkService`.
+		self.num_connected.store(self.network_service.user_protocol_mut().num_connected_peers(), Ordering::Relaxed);
+		{
+			let external_addresses = Swarm::<B, S, H>::external_addresses(&self.network_service).cloned().collect();
+			*self.external_addresses.lock() = external_addresses;
+		}
+		self.is_major_syncing.store(match self.network_service.user_protocol_mut().sync_state() {
+			SyncState::Idle => false,
+			SyncState::Downloading => true,
+		}, Ordering::Relaxed);
+
+		Ok(Async::NotReady)
+	}
+}
+
+/// The libp2p swarm, customized for our needs.
+type Swarm<B, S, H> = libp2p::swarm::Swarm<
+	Boxed<(PeerId, StreamMuxerBox), io::Error>,
+	Behaviour<B, S, H>
+>;
+
+// Implementation of `import_queue::Link` trait using the available local variables.
+struct NetworkLink<'a, B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> {
+	protocol: &'a mut Swarm<B, S, H>,
+}
+
+impl<'a, B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Link<B> for NetworkLink<'a, B, S, H> {
+	fn blocks_processed(
+		&mut self,
+		imported: usize,
+		count: usize,
+		results: Vec<(Result<BlockImportResult<NumberFor<B>>, BlockImportError>, B::Hash)>
+	) {
+		self.protocol.user_protocol_mut().blocks_processed(imported, count, results)
+	}
+	fn justification_imported(&mut self, who: PeerId, hash: &B::Hash, number: NumberFor<B>, success: bool) {
+		self.protocol.user_protocol_mut().justification_import_result(hash.clone(), number, success);
+		if !success {
+			info!("Invalid justification provided by {} for #{}", who, hash);
+			self.protocol.user_protocol_mut().disconnect_peer(&who);
+			self.protocol.user_protocol_mut().report_peer(who, i32::min_value());
+		}
+	}
+	fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		self.protocol.user_protocol_mut().request_justification(hash, number)
+	}
+	fn request_finality_proof(&mut self, hash: &B::Hash, number: NumberFor<B>) {
+		self.protocol.user_protocol_mut().request_finality_proof(hash, number)
+	}
+	fn finality_proof_imported(
+		&mut self,
+		who: PeerId,
+		request_block: (B::Hash, NumberFor<B>),
+		finalization_result: Result<(B::Hash, NumberFor<B>), ()>,
+	) {
+		let success = finalization_result.is_ok();
+		self.protocol.user_protocol_mut().finality_proof_import_result(request_block, finalization_result);
+		if !success {
+			info!("Invalid finality proof provided by {} for #{}", who, request_block.0);
+			self.protocol.user_protocol_mut().disconnect_peer(&who);
+			self.protocol.user_protocol_mut().report_peer(who, i32::min_value());
+		}
+	}
+}

substrate/client/network/src/test/block_import.rs

@@ -0,0 +1,88 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Testing block import logic.
+
+use consensus::ImportedAux;
+use consensus::import_queue::{
+	import_single_block, BasicQueue, BlockImportError, BlockImportResult, IncomingBlock,
+};
+use test_client::{self, prelude::*};
+use test_client::runtime::{Block, Hash};
+use sr_primitives::generic::BlockId;
+use super::*;
+
+fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock<Block>) {
+	let client = test_client::new();
+	let block = client.new_block(Default::default()).unwrap().bake().unwrap();
+	client.import(BlockOrigin::File, block).unwrap();
+
+	let (hash, number) = (client.block_hash(1).unwrap().unwrap(), 1);
+	let header = client.header(&BlockId::Number(1)).unwrap();
+	let justification = client.justification(&BlockId::Number(1)).unwrap();
+	let peer_id = PeerId::random();
+	(client, hash, number, peer_id.clone(), IncomingBlock {
+		hash,
+		header,
+		body: Some(Vec::new()),
+		justification,
+		origin: Some(peer_id.clone()),
+		allow_missing_state: false,
+	})
+}
+
+#[test]
+fn import_single_good_block_works() {
+	let (_, _hash, number, peer_id, block) = prepare_good_block();
+
+	let mut expected_aux = ImportedAux::default();
+	expected_aux.is_new_best = true;
+
+	match import_single_block(&mut test_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier(true)) {
+		Ok(BlockImportResult::ImportedUnknown(ref num, ref aux, ref org))
+			if *num == number && *aux == expected_aux && *org == Some(peer_id) => {}
+		r @ _ => panic!("{:?}", r)
+	}
+}
+
+#[test]
+fn import_single_good_known_block_is_ignored() {
+	let (mut client, _hash, number, _, block) = prepare_good_block();
+	match import_single_block(&mut client, BlockOrigin::File, block, &mut PassThroughVerifier(true)) {
+		Ok(BlockImportResult::ImportedKnown(ref n)) if *n == number => {}
+		_ => panic!()
+	}
+}
+
+#[test]
+fn import_single_good_block_without_header_fails() {
+	let (_, _, _, peer_id, mut block) = prepare_good_block();
+	block.header = None;
+	match import_single_block(&mut test_client::new(), BlockOrigin::File, block, &mut PassThroughVerifier(true)) {
+		Err(BlockImportError::IncompleteHeader(ref org)) if *org == Some(peer_id) => {}
+		_ => panic!()
+	}
+}
+
+#[test]
+fn async_import_queue_drops() {
+	// Perform this test multiple times since it exhibits non-deterministic behavior.
+	for _ in 0..100 {
+		let verifier = PassThroughVerifier(true);
+		let queue = BasicQueue::new(verifier, Box::new(test_client::new()), None, None);
+		drop(queue);
+	}
+}

substrate/client/network/src/test/mod.rs

@@ -0,0 +1,823 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+#![allow(missing_docs)]
+
+#[cfg(test)]
+mod block_import;
+#[cfg(test)]
+mod sync;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::config::build_multiaddr;
+use log::trace;
+use crate::chain::FinalityProofProvider;
+use client_api::{
+	ClientInfo, BlockchainEvents, BlockImportNotification,
+	FinalityNotifications, ImportNotifications,
+	FinalityNotification,
+	error::Result as ClientResult,
+	well_known_cache_keys::{self, Id as CacheKeyId},
+	backend::{AuxStore, Backend, Finalizer}
+};
+use block_builder::BlockBuilder;
+use client::LongestChain;
+use crate::config::Roles;
+use consensus::block_validation::DefaultBlockAnnounceValidator;
+use consensus::import_queue::BasicQueue;
+use consensus::import_queue::{
+	BoxBlockImport, BoxJustificationImport, Verifier, BoxFinalityProofImport,
+};
+use consensus::block_import::{BlockImport, ImportResult};
+use consensus::Error as ConsensusError;
+use consensus::{BlockOrigin, ForkChoiceStrategy, BlockImportParams, BlockCheckParams, JustificationImport};
+use futures::prelude::*;
+use futures03::{StreamExt as _, TryStreamExt as _};
+use crate::{NetworkWorker, NetworkService, ReportHandle, config::ProtocolId};
+use crate::config::{NetworkConfiguration, TransportConfig, BoxFinalityProofRequestBuilder};
+use libp2p::PeerId;
+use parking_lot::Mutex;
+use primitives::H256;
+use crate::protocol::{Context, ProtocolConfig};
+use sr_primitives::generic::{BlockId, OpaqueDigestItemId};
+use sr_primitives::traits::{Block as BlockT, Header, NumberFor};
+use sr_primitives::Justification;
+use crate::service::TransactionPool;
+use crate::specialization::NetworkSpecialization;
+use test_client::{self, AccountKeyring};
+
+pub use test_client::runtime::{Block, Extrinsic, Hash, Transfer};
+pub use test_client::{TestClient, TestClientBuilder, TestClientBuilderExt};
+
+type AuthorityId = babe_primitives::AuthorityId;
+
+#[cfg(any(test, feature = "test-helpers"))]
+/// A Verifier that accepts all blocks and passes them on with the configured
+/// finality to be imported.
+#[derive(Clone)]
+pub struct PassThroughVerifier(pub bool);
+
+#[cfg(any(test, feature = "test-helpers"))]
+/// This `Verifier` accepts all data as valid.
+impl<B: BlockT> Verifier<B> for PassThroughVerifier {
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		header: B::Header,
+		justification: Option<Justification>,
+		body: Option<Vec<B::Extrinsic>>
+	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		let maybe_keys = header.digest()
+			.log(|l| l.try_as_raw(OpaqueDigestItemId::Consensus(b"aura"))
+				.or_else(|| l.try_as_raw(OpaqueDigestItemId::Consensus(b"babe")))
+			)
+			.map(|blob| vec![(well_known_cache_keys::AUTHORITIES, blob.to_vec())]);
+
+		Ok((BlockImportParams {
+			origin,
+			header,
+			body,
+			finalized: self.0,
+			justification,
+			post_digests: vec![],
+			auxiliary: Vec::new(),
+			fork_choice: ForkChoiceStrategy::LongestChain,
+			allow_missing_state: false,
+		}, maybe_keys))
+	}
+}
+
+/// The test specialization.
+#[derive(Clone)]
+pub struct DummySpecialization;
+
+impl NetworkSpecialization<Block> for DummySpecialization {
+	fn status(&self) -> Vec<u8> {
+		vec![]
+	}
+
+	fn on_connect(
+		&mut self,
+		_ctx: &mut dyn Context<Block>,
+		_peer_id: PeerId,
+		_status: crate::message::Status<Block>
+	) {}
+
+	fn on_disconnect(&mut self, _ctx: &mut dyn Context<Block>, _peer_id: PeerId) {}
+
+	fn on_message(
+		&mut self,
+		_ctx: &mut dyn Context<Block>,
+		_peer_id: PeerId,
+		_message: Vec<u8>,
+	) {}
+}
+
+pub type PeersFullClient =
+	client::Client<test_client::Backend, test_client::Executor, Block, test_client::runtime::RuntimeApi>;
+pub type PeersLightClient =
+	client::Client<test_client::LightBackend, test_client::LightExecutor, Block, test_client::runtime::RuntimeApi>;
+
+#[derive(Clone)]
+pub enum PeersClient {
+	Full(Arc<PeersFullClient>, Arc<test_client::Backend>),
+	Light(Arc<PeersLightClient>, Arc<test_client::LightBackend>),
+}
+
+impl PeersClient {
+	pub fn as_full(&self) -> Option<Arc<PeersFullClient>> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => Some(client.clone()),
+			_ => None,
+		}
+	}
+
+	pub fn as_block_import(&self) -> BoxBlockImport<Block> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => Box::new(client.clone()) as _,
+			PeersClient::Light(ref client, ref _backend) => Box::new(client.clone()) as _,
+		}
+	}
+
+	pub fn get_aux(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.get_aux(key),
+			PeersClient::Light(ref client, ref _backend) => client.get_aux(key),
+		}
+	}
+
+	pub fn info(&self) -> ClientInfo<Block> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.info(),
+			PeersClient::Light(ref client, ref _backend) => client.info(),
+		}
+	}
+
+	pub fn header(&self, block: &BlockId<Block>) -> ClientResult<Option<<Block as BlockT>::Header>> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.header(block),
+			PeersClient::Light(ref client, ref _backend) => client.header(block),
+		}
+	}
+
+	pub fn justification(&self, block: &BlockId<Block>) -> ClientResult<Option<Justification>> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.justification(block),
+			PeersClient::Light(ref client, ref _backend) => client.justification(block),
+		}
+	}
+
+	pub fn finality_notification_stream(&self) -> FinalityNotifications<Block> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.finality_notification_stream(),
+			PeersClient::Light(ref client, ref _backend) => client.finality_notification_stream(),
+		}
+	}
+
+	pub fn import_notification_stream(&self) -> ImportNotifications<Block>{
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.import_notification_stream(),
+			PeersClient::Light(ref client, ref _backend) => client.import_notification_stream(),
+		}
+	}
+
+	pub fn finalize_block(
+		&self,
+		id: BlockId<Block>,
+		justification: Option<Justification>,
+		notify: bool
+	) -> ClientResult<()> {
+		match *self {
+			PeersClient::Full(ref client, ref _backend) => client.finalize_block(id, justification, notify),
+			PeersClient::Light(ref client, ref _backend) => client.finalize_block(id, justification, notify),
+		}
+	}
+}
+
+pub struct Peer<D, S: NetworkSpecialization<Block>> {
+	pub data: D,
+	client: PeersClient,
+	/// We keep a copy of the verifier so that we can invoke it for locally-generated blocks,
+	/// instead of going through the import queue.
+	verifier: VerifierAdapter<dyn Verifier<Block>>,
+	/// We keep a copy of the block_import so that we can invoke it for locally-generated blocks,
+	/// instead of going through the import queue.
+	block_import: Box<dyn BlockImport<Block, Error = ConsensusError>>,
+	select_chain: Option<LongestChain<test_client::Backend, Block>>,
+	backend: Option<Arc<test_client::Backend>>,
+	network: NetworkWorker<Block, S, <Block as BlockT>::Hash>,
+	imported_blocks_stream: Box<dyn Stream<Item = BlockImportNotification<Block>, Error = ()> + Send>,
+	finality_notification_stream: Box<dyn Stream<Item = FinalityNotification<Block>, Error = ()> + Send>,
+}
+
+impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
+	/// Get this peer ID.
+	pub fn id(&self) -> PeerId {
+		self.network.service().local_peer_id()
+	}
+
+	/// Returns true if we're major syncing.
+	pub fn is_major_syncing(&self) -> bool {
+		self.network.service().is_major_syncing()
+	}
+
+	// Returns a clone of the local SelectChain, only available on full nodes
+	pub fn select_chain(&self) -> Option<LongestChain<test_client::Backend, Block>> {
+		self.select_chain.clone()
+	}
+
+	/// Returns the number of peers we're connected to.
+	pub fn num_peers(&self) -> usize {
+		self.network.num_connected_peers()
+	}
+
+	/// Returns true if we have no peer.
+	pub fn is_offline(&self) -> bool {
+		self.num_peers() == 0
+	}
+
+	/// Request a justification for the given block.
+	pub fn request_justification(&self, hash: &<Block as BlockT>::Hash, number: NumberFor<Block>) {
+		self.network.service().request_justification(hash, number);
+	}
+
+	/// Announces an important block on the network.
+	pub fn announce_block(&self, hash: <Block as BlockT>::Hash, data: Vec<u8>) {
+		self.network.service().announce_block(hash, data);
+	}
+
+	/// Request explicit fork sync.
+	pub fn set_sync_fork_request(&self, peers: Vec<PeerId>, hash: <Block as BlockT>::Hash, number: NumberFor<Block>) {
+		self.network.service().set_sync_fork_request(peers, hash, number);
+	}
+
+	/// Add blocks to the peer -- edit the block before adding
+	pub fn generate_blocks<F>(&mut self, count: usize, origin: BlockOrigin, edit_block: F) -> H256
+		where F: FnMut(BlockBuilder<Block, PeersFullClient>) -> Block
+	{
+		let best_hash = self.client.info().chain.best_hash;
+		self.generate_blocks_at(BlockId::Hash(best_hash), count, origin, edit_block)
+	}
+
+	/// Add blocks to the peer -- edit the block before adding. The chain will
+	/// start at the given block iD.
+	fn generate_blocks_at<F>(
+		&mut self,
+		at: BlockId<Block>,
+		count: usize,
+		origin: BlockOrigin,
+		mut edit_block: F
+	) -> H256 where F: FnMut(BlockBuilder<Block, PeersFullClient>) -> Block {
+		let full_client = self.client.as_full().expect("blocks could only be generated by full clients");
+		let mut at = full_client.header(&at).unwrap().unwrap().hash();
+		for _  in 0..count {
+			let builder = full_client.new_block_at(&BlockId::Hash(at), Default::default()
+			).unwrap();
+			let block = edit_block(builder);
+			let hash = block.header.hash();
+			trace!(
+				target: "test_network",
+				"Generating {}, (#{}, parent={})",
+				hash,
+				block.header.number,
+				block.header.parent_hash
+			);
+			let header = block.header.clone();
+			let (import_block, cache) = self.verifier.verify(
+				origin,
+				header.clone(),
+				None,
+				Some(block.extrinsics)
+			).unwrap();
+			let cache = if let Some(cache) = cache {
+				cache.into_iter().collect()
+			} else {
+				Default::default()
+			};
+			self.block_import.import_block(import_block, cache).expect("block_import failed");
+			self.network.on_block_imported(hash, header, Vec::new(), true);
+			at = hash;
+		}
+
+		self.network.service().announce_block(at.clone(), Vec::new());
+		at
+	}
+
+	/// Push blocks to the peer (simplified: with or without a TX)
+	pub fn push_blocks(&mut self, count: usize, with_tx: bool) -> H256 {
+		let best_hash = self.client.info().chain.best_hash;
+		self.push_blocks_at(BlockId::Hash(best_hash), count, with_tx)
+	}
+
+	/// Push blocks to the peer (simplified: with or without a TX) starting from
+	/// given hash.
+	pub fn push_blocks_at(&mut self, at: BlockId<Block>, count: usize, with_tx: bool) -> H256 {
+		let mut nonce = 0;
+		if with_tx {
+			self.generate_blocks_at(at, count, BlockOrigin::File, |mut builder| {
+				let transfer = Transfer {
+					from: AccountKeyring::Alice.into(),
+					to: AccountKeyring::Alice.into(),
+					amount: 1,
+					nonce,
+				};
+				builder.push(transfer.into_signed_tx()).unwrap();
+				nonce = nonce + 1;
+				builder.bake().unwrap()
+			})
+		} else {
+			self.generate_blocks_at(at, count, BlockOrigin::File, |builder| builder.bake().unwrap())
+		}
+	}
+
+	pub fn push_authorities_change_block(&mut self, new_authorities: Vec<AuthorityId>) -> H256 {
+		self.generate_blocks(1, BlockOrigin::File, |mut builder| {
+			builder.push(Extrinsic::AuthoritiesChange(new_authorities.clone())).unwrap();
+			builder.bake().unwrap()
+		})
+	}
+
+	/// Get a reference to the client.
+	pub fn client(&self) -> &PeersClient {
+		&self.client
+	}
+
+	/// Get a reference to the network service.
+	pub fn network_service(&self) -> &Arc<NetworkService<Block, S, <Block as BlockT>::Hash>> {
+		&self.network.service()
+	}
+
+	/// Test helper to compare the blockchain state of multiple (networked)
+	/// clients.
+	/// Potentially costly, as it creates in-memory copies of both blockchains in order
+	/// to compare them. If you have easier/softer checks that are sufficient, e.g.
+	/// by using .info(), you should probably use it instead of this.
+	pub fn blockchain_canon_equals(&self, other: &Self) -> bool {
+		if let (Some(mine), Some(others)) = (self.backend.clone(), other.backend.clone()) {
+			mine.as_in_memory().blockchain()
+				.canon_equals_to(others.as_in_memory().blockchain())
+		} else {
+			false
+		}
+	}
+
+	/// Count the total number of imported blocks.
+	pub fn blocks_count(&self) -> u64 {
+		self.backend.as_ref().map(
+			|backend| backend.blocks_count()
+		).unwrap_or(0)
+	}
+}
+
+pub struct EmptyTransactionPool;
+
+impl TransactionPool<Hash, Block> for EmptyTransactionPool {
+	fn transactions(&self) -> Vec<(Hash, Extrinsic)> {
+		Vec::new()
+	}
+
+	fn hash_of(&self, _transaction: &Extrinsic) -> Hash {
+		Hash::default()
+	}
+
+	fn import(
+		&self,
+		_report_handle: ReportHandle,
+		_who: PeerId,
+		_rep_change_good: i32,
+		_rep_change_bad: i32,
+		_transaction: Extrinsic
+	) {}
+
+	fn on_broadcasted(&self, _: HashMap<Hash, Vec<String>>) {}
+}
+
+pub trait SpecializationFactory {
+	fn create() -> Self;
+}
+
+impl SpecializationFactory for DummySpecialization {
+	fn create() -> DummySpecialization {
+		DummySpecialization
+	}
+}
+
+/// Implements `BlockImport` on an `Arc<Mutex<impl BlockImport>>`. Used internally. Necessary to overcome the way the
+/// `TestNet` trait is designed, more specifically `make_block_import` returning a `Box<BlockImport>` makes it
+/// impossible to clone the underlying object.
+struct BlockImportAdapter<T: ?Sized>(Arc<Mutex<Box<T>>>);
+
+impl<T: ?Sized> Clone for BlockImportAdapter<T> {
+	fn clone(&self) -> Self {
+		BlockImportAdapter(self.0.clone())
+	}
+}
+
+impl<T: ?Sized + BlockImport<Block>> BlockImport<Block> for BlockImportAdapter<T> {
+	type Error = T::Error;
+
+	fn check_block(
+		&mut self,
+		block: BlockCheckParams<Block>,
+	) -> Result<ImportResult, Self::Error> {
+		self.0.lock().check_block(block)
+	}
+
+	fn import_block(
+		&mut self,
+		block: BlockImportParams<Block>,
+		cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+	) -> Result<ImportResult, Self::Error> {
+		self.0.lock().import_block(block, cache)
+	}
+}
+
+/// Implements `Verifier` on an `Arc<Mutex<impl Verifier>>`. Used internally.
+struct VerifierAdapter<T: ?Sized>(Arc<Mutex<Box<T>>>);
+
+impl<T: ?Sized> Clone for VerifierAdapter<T> {
+	fn clone(&self) -> Self {
+		VerifierAdapter(self.0.clone())
+	}
+}
+
+impl<B: BlockT, T: ?Sized + Verifier<B>> Verifier<B> for VerifierAdapter<T> {
+	fn verify(
+		&mut self,
+		origin: BlockOrigin,
+		header: B::Header,
+		justification: Option<Justification>,
+		body: Option<Vec<B::Extrinsic>>
+	) -> Result<(BlockImportParams<B>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
+		self.0.lock().verify(origin, header, justification, body)
+	}
+}
+
+pub trait TestNetFactory: Sized {
+	type Specialization: NetworkSpecialization<Block> + SpecializationFactory;
+	type Verifier: 'static + Verifier<Block>;
+	type PeerData: Default;
+
+	/// These two need to be implemented!
+	fn from_config(config: &ProtocolConfig) -> Self;
+	fn make_verifier(
+		&self,
+		client: PeersClient,
+		config: &ProtocolConfig,
+		peer_data: &Self::PeerData,
+	) -> Self::Verifier;
+
+	/// Get reference to peer.
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::Specialization>;
+	fn peers(&self) -> &Vec<Peer<Self::PeerData, Self::Specialization>>;
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(&mut self, closure: F);
+
+	/// Get custom block import handle for fresh client, along with peer data.
+	fn make_block_import(&self, client: PeersClient)
+		-> (
+			BoxBlockImport<Block>,
+			Option<BoxJustificationImport<Block>>,
+			Option<BoxFinalityProofImport<Block>>,
+			Option<BoxFinalityProofRequestBuilder<Block>>,
+			Self::PeerData,
+		)
+	{
+		(client.as_block_import(), None, None, None, Default::default())
+	}
+
+	/// Get finality proof provider (if supported).
+	fn make_finality_proof_provider(&self, _client: PeersClient) -> Option<Arc<dyn FinalityProofProvider<Block>>> {
+		None
+	}
+
+	fn default_config() -> ProtocolConfig {
+		ProtocolConfig::default()
+	}
+
+	/// Create new test network with this many peers.
+	fn new(n: usize) -> Self {
+		trace!(target: "test_network", "Creating test network");
+		let config = Self::default_config();
+		let mut net = Self::from_config(&config);
+
+		for i in 0..n {
+			trace!(target: "test_network", "Adding peer {}", i);
+			net.add_full_peer(&config);
+		}
+		net
+	}
+
+	fn add_full_peer(&mut self, config: &ProtocolConfig) {
+		self.add_full_peer_with_states(config, None)
+	}
+
+	/// Add a full peer.
+	fn add_full_peer_with_states(&mut self, config: &ProtocolConfig, keep_blocks: Option<u32>) {
+		let test_client_builder = match keep_blocks {
+			Some(keep_blocks) => TestClientBuilder::with_pruning_window(keep_blocks),
+			None => TestClientBuilder::with_default_backend(),
+		};
+		let backend = test_client_builder.backend();
+		let (c, longest_chain) = test_client_builder.build_with_longest_chain();
+		let client = Arc::new(c);
+
+		let (
+			block_import,
+			justification_import,
+			finality_proof_import,
+			finality_proof_request_builder,
+			data,
+		) = self.make_block_import(PeersClient::Full(client.clone(), backend.clone()));
+		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));
+
+		let verifier = self.make_verifier(
+			PeersClient::Full(client.clone(), backend.clone()),
+			config,
+			&data,
+		);
+		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
+
+		let import_queue = Box::new(BasicQueue::new(
+			verifier.clone(),
+			Box::new(block_import.clone()),
+			justification_import,
+			finality_proof_import,
+		));
+
+		let listen_addr = build_multiaddr![Memory(rand::random::<u64>())];
+
+		let network = NetworkWorker::new(crate::config::Params {
+			roles: config.roles,
+			network_config: NetworkConfiguration {
+				listen_addresses: vec![listen_addr.clone()],
+				transport: TransportConfig::MemoryOnly,
+				..NetworkConfiguration::default()
+			},
+			chain: client.clone(),
+			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Full(client.clone(), backend.clone())),
+			finality_proof_request_builder,
+			on_demand: None,
+			transaction_pool: Arc::new(EmptyTransactionPool),
+			protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
+			import_queue,
+			specialization: self::SpecializationFactory::create(),
+			block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone()))
+		}).unwrap();
+
+		self.mut_peers(|peers| {
+			for peer in peers.iter_mut() {
+				peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone());
+			}
+
+			let imported_blocks_stream = Box::new(client.import_notification_stream()
+				.map(|v| Ok::<_, ()>(v)).compat().fuse());
+			let finality_notification_stream = Box::new(client.finality_notification_stream()
+				.map(|v| Ok::<_, ()>(v)).compat().fuse());
+
+			peers.push(Peer {
+				data,
+				client: PeersClient::Full(client, backend.clone()),
+				select_chain: Some(longest_chain),
+				backend: Some(backend),
+				imported_blocks_stream,
+				finality_notification_stream,
+				block_import: Box::new(block_import),
+				verifier,
+				network,
+			});
+		});
+	}
+
+	/// Add a light peer.
+	fn add_light_peer(&mut self, config: &ProtocolConfig) {
+		let mut config = config.clone();
+		config.roles = Roles::LIGHT;
+
+		let (c, backend) = test_client::new_light();
+		let client = Arc::new(c);
+		let (
+			block_import,
+			justification_import,
+			finality_proof_import,
+			finality_proof_request_builder,
+			data,
+		) = self.make_block_import(PeersClient::Light(client.clone(), backend.clone()));
+		let block_import = BlockImportAdapter(Arc::new(Mutex::new(block_import)));
+
+		let verifier = self.make_verifier(
+			PeersClient::Light(client.clone(), backend.clone()),
+			&config,
+			&data,
+		);
+		let verifier = VerifierAdapter(Arc::new(Mutex::new(Box::new(verifier) as Box<_>)));
+
+		let import_queue = Box::new(BasicQueue::new(
+			verifier.clone(),
+			Box::new(block_import.clone()),
+			justification_import,
+			finality_proof_import,
+		));
+
+		let listen_addr = build_multiaddr![Memory(rand::random::<u64>())];
+
+		let network = NetworkWorker::new(crate::config::Params {
+			roles: config.roles,
+			network_config: NetworkConfiguration {
+				listen_addresses: vec![listen_addr.clone()],
+				transport: TransportConfig::MemoryOnly,
+				..NetworkConfiguration::default()
+			},
+			chain: client.clone(),
+			finality_proof_provider: self.make_finality_proof_provider(PeersClient::Light(client.clone(), backend.clone())),
+			finality_proof_request_builder,
+			on_demand: None,
+			transaction_pool: Arc::new(EmptyTransactionPool),
+			protocol_id: ProtocolId::from(&b"test-protocol-name"[..]),
+			import_queue,
+			specialization: self::SpecializationFactory::create(),
+			block_announce_validator: Box::new(DefaultBlockAnnounceValidator::new(client.clone()))
+		}).unwrap();
+
+		self.mut_peers(|peers| {
+			for peer in peers.iter_mut() {
+				peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone());
+			}
+
+			let imported_blocks_stream = Box::new(client.import_notification_stream()
+				.map(|v| Ok::<_, ()>(v)).compat().fuse());
+			let finality_notification_stream = Box::new(client.finality_notification_stream()
+				.map(|v| Ok::<_, ()>(v)).compat().fuse());
+
+			peers.push(Peer {
+				data,
+				verifier,
+				select_chain: None,
+				backend: None,
+				block_import: Box::new(block_import),
+				client: PeersClient::Light(client, backend),
+				imported_blocks_stream,
+				finality_notification_stream,
+				network,
+			});
+		});
+	}
+
+	/// Polls the testnet until all nodes are in sync.
+	///
+	/// Must be executed in a task context.
+	fn poll_until_sync(&mut self) -> Async<()> {
+		self.poll();
+
+		// Return `NotReady` if there's a mismatch in the highest block number.
+		let mut highest = None;
+		for peer in self.peers().iter() {
+			if peer.is_major_syncing() || peer.network.num_queued_blocks() != 0 {
+				return Async::NotReady
+			}
+			match (highest, peer.client.info().chain.best_hash) {
+				(None, b) => highest = Some(b),
+				(Some(ref a), ref b) if a == b => {},
+				(Some(_), _) => return Async::NotReady,
+			}
+		}
+		Async::Ready(())
+	}
+
+	/// Blocks the current thread until we are sync'ed.
+	///
+	/// Calls `poll_until_sync` repeatidely with the runtime passed as parameter.
+	fn block_until_sync(&mut self, runtime: &mut tokio::runtime::current_thread::Runtime) {
+		runtime.block_on(futures::future::poll_fn::<(), (), _>(|| Ok(self.poll_until_sync()))).unwrap();
+	}
+
+	/// Polls the testnet. Processes all the pending actions and returns `NotReady`.
+	fn poll(&mut self) {
+		self.mut_peers(|peers| {
+			for peer in peers {
+				trace!(target: "sync", "-- Polling {}", peer.id());
+				peer.network.poll().unwrap();
+				trace!(target: "sync", "-- Polling complete {}", peer.id());
+
+				// We poll `imported_blocks_stream`.
+				while let Ok(Async::Ready(Some(notification))) = peer.imported_blocks_stream.poll() {
+					peer.network.on_block_imported(notification.hash, notification.header, Vec::new(), true);
+				}
+
+				// We poll `finality_notification_stream`, but we only take the last event.
+				let mut last = None;
+				while let Ok(Async::Ready(Some(item))) = peer.finality_notification_stream.poll() {
+					last = Some(item);
+				}
+				if let Some(notification) = last {
+					peer.network.on_block_finalized(notification.hash, notification.header);
+				}
+			}
+		});
+	}
+}
+
+pub struct TestNet {
+	peers: Vec<Peer<(), DummySpecialization>>,
+}
+
+impl TestNetFactory for TestNet {
+	type Specialization = DummySpecialization;
+	type Verifier = PassThroughVerifier;
+	type PeerData = ();
+
+	/// Create new test network with peers and given config.
+	fn from_config(_config: &ProtocolConfig) -> Self {
+		TestNet {
+			peers: Vec::new(),
+		}
+	}
+
+	fn make_verifier(&self, _client: PeersClient, _config: &ProtocolConfig, _peer_data: &())
+		-> Self::Verifier
+	{
+		PassThroughVerifier(false)
+	}
+
+	fn peer(&mut self, i: usize) -> &mut Peer<(), Self::Specialization> {
+		&mut self.peers[i]
+	}
+
+	fn peers(&self) -> &Vec<Peer<(), Self::Specialization>> {
+		&self.peers
+	}
+
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<(), Self::Specialization>>)>(&mut self, closure: F) {
+		closure(&mut self.peers);
+	}
+}
+
+pub struct ForceFinalized(PeersClient);
+
+impl JustificationImport<Block> for ForceFinalized {
+	type Error = ConsensusError;
+
+	fn import_justification(
+		&mut self,
+		hash: H256,
+		_number: NumberFor<Block>,
+		justification: Justification,
+	) -> Result<(), Self::Error> {
+		self.0.finalize_block(BlockId::Hash(hash), Some(justification), true)
+			.map_err(|_| ConsensusError::InvalidJustification.into())
+	}
+}
+
+pub struct JustificationTestNet(TestNet);
+
+impl TestNetFactory for JustificationTestNet {
+	type Specialization = DummySpecialization;
+	type Verifier = PassThroughVerifier;
+	type PeerData = ();
+
+	fn from_config(config: &ProtocolConfig) -> Self {
+		JustificationTestNet(TestNet::from_config(config))
+	}
+
+	fn make_verifier(&self, client: PeersClient, config: &ProtocolConfig, peer_data: &()) -> Self::Verifier {
+		self.0.make_verifier(client, config, peer_data)
+	}
+
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::Specialization> {
+		self.0.peer(i)
+	}
+
+	fn peers(&self) -> &Vec<Peer<Self::PeerData, Self::Specialization>> {
+		self.0.peers()
+	}
+
+	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(&mut self, closure: F) {
+		self.0.mut_peers(closure)
+	}
+
+	fn make_block_import(&self, client: PeersClient)
+		-> (
+			BoxBlockImport<Block>,
+			Option<BoxJustificationImport<Block>>,
+			Option<BoxFinalityProofImport<Block>>,
+			Option<BoxFinalityProofRequestBuilder<Block>>,
+			Self::PeerData,
+		)
+	{
+		(client.as_block_import(), Some(Box::new(ForceFinalized(client))), None, None, Default::default())
+	}
+}

substrate/client/network/src/test/sync.rs

@@ -0,0 +1,662 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::config::Roles;
+use consensus::BlockOrigin;
+use futures03::TryFutureExt as _;
+use std::time::Duration;
+use tokio::runtime::current_thread;
+use super::*;
+
+fn test_ancestor_search_when_common_is(n: usize) {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	net.peer(0).push_blocks(n, false);
+	net.peer(1).push_blocks(n, false);
+	net.peer(2).push_blocks(n, false);
+
+	net.peer(0).push_blocks(10, true);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
+
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn sync_peers_works() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		for peer in 0..3 {
+			if net.peer(peer).num_peers() != 2 {
+				return Ok(Async::NotReady)
+			}
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+}
+
+#[test]
+fn sync_cycle_from_offline_to_syncing_to_offline() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	for peer in 0..3 {
+		// Offline, and not major syncing.
+		assert!(net.peer(peer).is_offline());
+		assert!(!net.peer(peer).is_major_syncing());
+	}
+
+	// Generate blocks.
+	net.peer(2).push_blocks(100, false);
+
+	// Block until all nodes are online and nodes 0 and 1 and major syncing.
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		for peer in 0..3 {
+			// Online
+			if net.peer(peer).is_offline() {
+				return Ok(Async::NotReady)
+			}
+			if peer < 2 {
+				// Major syncing.
+				if !net.peer(peer).is_major_syncing() {
+					return Ok(Async::NotReady)
+				}
+			}
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+
+	// Block until all nodes are done syncing.
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		for peer in 0..3 {
+			if net.peer(peer).is_major_syncing() {
+				return Ok(Async::NotReady)
+			}
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+
+	// Now drop nodes 1 and 2, and check that node 0 is offline.
+	net.peers.remove(2);
+	net.peers.remove(1);
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if !net.peer(0).is_offline() {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+}
+
+#[test]
+fn syncing_node_not_major_syncing_when_disconnected() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	// Generate blocks.
+	net.peer(2).push_blocks(100, false);
+
+	// Check that we're not major syncing when disconnected.
+	assert!(!net.peer(1).is_major_syncing());
+
+	// Check that we switch to major syncing.
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if !net.peer(1).is_major_syncing() {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+
+	// Destroy two nodes, and check that we switch to non-major syncing.
+	net.peers.remove(2);
+	net.peers.remove(0);
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(0).is_major_syncing() {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+}
+
+#[test]
+fn sync_from_two_peers_works() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+	assert!(!net.peer(0).is_major_syncing());
+}
+
+#[test]
+fn sync_from_two_peers_with_ancestry_search_works() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	net.peer(0).push_blocks(10, true);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn ancestry_search_works_when_backoff_is_one() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	net.peer(0).push_blocks(1, false);
+	net.peer(1).push_blocks(2, false);
+	net.peer(2).push_blocks(2, false);
+
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn ancestry_search_works_when_ancestor_is_genesis() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	net.peer(0).push_blocks(13, true);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
+
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn ancestry_search_works_when_common_is_one() {
+	test_ancestor_search_when_common_is(1);
+}
+
+#[test]
+fn ancestry_search_works_when_common_is_two() {
+	test_ancestor_search_when_common_is(2);
+}
+
+#[test]
+fn ancestry_search_works_when_common_is_hundred() {
+	test_ancestor_search_when_common_is(100);
+}
+
+#[test]
+fn sync_long_chain_works() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(2);
+	net.peer(1).push_blocks(500, false);
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn sync_no_common_longer_chain_fails() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	net.peer(0).push_blocks(20, true);
+	net.peer(1).push_blocks(20, false);
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(0).is_major_syncing() {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+	let peer1 = &net.peers()[1];
+	assert!(!net.peers()[0].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn sync_justifications() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = JustificationTestNet::new(3);
+	net.peer(0).push_blocks(20, false);
+	net.block_until_sync(&mut runtime);
+
+	// there's currently no justification for block #10
+	assert_eq!(net.peer(0).client().justification(&BlockId::Number(10)).unwrap(), None);
+	assert_eq!(net.peer(1).client().justification(&BlockId::Number(10)).unwrap(), None);
+
+	// we finalize block #10, #15 and #20 for peer 0 with a justification
+	net.peer(0).client().finalize_block(BlockId::Number(10), Some(Vec::new()), true).unwrap();
+	net.peer(0).client().finalize_block(BlockId::Number(15), Some(Vec::new()), true).unwrap();
+	net.peer(0).client().finalize_block(BlockId::Number(20), Some(Vec::new()), true).unwrap();
+
+	let h1 = net.peer(1).client().header(&BlockId::Number(10)).unwrap().unwrap();
+	let h2 = net.peer(1).client().header(&BlockId::Number(15)).unwrap().unwrap();
+	let h3 = net.peer(1).client().header(&BlockId::Number(20)).unwrap().unwrap();
+
+	// peer 1 should get the justifications from the network
+	net.peer(1).request_justification(&h1.hash().into(), 10);
+	net.peer(1).request_justification(&h2.hash().into(), 15);
+	net.peer(1).request_justification(&h3.hash().into(), 20);
+
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| {
+		net.poll();
+
+		for height in (10..21).step_by(5) {
+			if net.peer(0).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) {
+				return Ok(Async::NotReady);
+			}
+			if net.peer(1).client().justification(&BlockId::Number(height)).unwrap() != Some(Vec::new()) {
+				return Ok(Async::NotReady);
+			}
+		}
+
+		Ok(Async::Ready(()))
+	})).unwrap();
+}
+
+#[test]
+fn sync_justifications_across_forks() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = JustificationTestNet::new(3);
+	// we push 5 blocks
+	net.peer(0).push_blocks(5, false);
+	// and then two forks 5 and 6 blocks long
+	let f1_best = net.peer(0).push_blocks_at(BlockId::Number(5), 5, false);
+	let f2_best = net.peer(0).push_blocks_at(BlockId::Number(5), 6, false);
+
+	// peer 1 will only see the longer fork. but we'll request justifications
+	// for both and finalize the small fork instead.
+	net.block_until_sync(&mut runtime);
+
+	net.peer(0).client().finalize_block(BlockId::Hash(f1_best), Some(Vec::new()), true).unwrap();
+
+	net.peer(1).request_justification(&f1_best, 10);
+	net.peer(1).request_justification(&f2_best, 11);
+
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| {
+		net.poll();
+
+		if net.peer(0).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new()) &&
+			net.peer(1).client().justification(&BlockId::Number(10)).unwrap() == Some(Vec::new())
+		{
+			Ok(Async::Ready(()))
+		} else {
+			Ok(Async::NotReady)
+		}
+	})).unwrap();
+}
+
+#[test]
+fn sync_after_fork_works() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	net.peer(0).push_blocks(30, false);
+	net.peer(1).push_blocks(30, false);
+	net.peer(2).push_blocks(30, false);
+
+	net.peer(0).push_blocks(10, true);
+	net.peer(1).push_blocks(20, false);
+	net.peer(2).push_blocks(20, false);
+
+	net.peer(1).push_blocks(10, true);
+	net.peer(2).push_blocks(1, false);
+
+	// peer 1 has the best chain
+	net.block_until_sync(&mut runtime);
+	let peer1 = &net.peers()[1];
+	assert!(net.peers()[0].blockchain_canon_equals(peer1));
+	(net.peers()[1].blockchain_canon_equals(peer1));
+	(net.peers()[2].blockchain_canon_equals(peer1));
+}
+
+#[test]
+fn syncs_all_forks() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(4);
+	net.peer(0).push_blocks(2, false);
+	net.peer(1).push_blocks(2, false);
+
+	net.peer(0).push_blocks(2, true);
+	net.peer(1).push_blocks(4, false);
+
+	net.block_until_sync(&mut runtime);
+	// Check that all peers have all of the blocks.
+	assert_eq!(9, net.peer(0).blocks_count());
+	assert_eq!(9, net.peer(1).blocks_count());
+}
+
+#[test]
+fn own_blocks_are_announced() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+	net.block_until_sync(&mut runtime); // connect'em
+	net.peer(0).generate_blocks(1, BlockOrigin::Own, |builder| builder.bake().unwrap());
+
+	net.block_until_sync(&mut runtime);
+
+	assert_eq!(net.peer(0).client.info().chain.best_number, 1);
+	assert_eq!(net.peer(1).client.info().chain.best_number, 1);
+	let peer0 = &net.peers()[0];
+	assert!(net.peers()[1].blockchain_canon_equals(peer0));
+	(net.peers()[2].blockchain_canon_equals(peer0));
+}
+
+#[test]
+fn blocks_are_not_announced_by_light_nodes() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(0);
+
+	// full peer0 is connected to light peer
+	// light peer1 is connected to full peer2
+	let mut light_config = ProtocolConfig::default();
+	light_config.roles = Roles::LIGHT;
+	net.add_full_peer(&ProtocolConfig::default());
+	net.add_light_peer(&light_config);
+
+	// Sync between 0 and 1.
+	net.peer(0).push_blocks(1, false);
+	assert_eq!(net.peer(0).client.info().chain.best_number, 1);
+	net.block_until_sync(&mut runtime);
+	assert_eq!(net.peer(1).client.info().chain.best_number, 1);
+
+	// Add another node and remove node 0.
+	net.add_full_peer(&ProtocolConfig::default());
+	net.peers.remove(0);
+
+	// Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together.
+	let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).compat();
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| {
+		net.poll();
+		delay.poll().map_err(|_| ())
+	})).unwrap();
+	assert_eq!(net.peer(1).client.info().chain.best_number, 0);
+}
+
+#[test]
+fn can_sync_small_non_best_forks() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(2);
+	net.peer(0).push_blocks(30, false);
+	net.peer(1).push_blocks(30, false);
+
+	// small fork + reorg on peer 1.
+	net.peer(0).push_blocks_at(BlockId::Number(30), 2, true);
+	let small_hash = net.peer(0).client().info().chain.best_hash;
+	net.peer(0).push_blocks_at(BlockId::Number(30), 10, false);
+	assert_eq!(net.peer(0).client().info().chain.best_number, 40);
+
+	// peer 1 only ever had the long fork.
+	net.peer(1).push_blocks(10, false);
+	assert_eq!(net.peer(1).client().info().chain.best_number, 40);
+
+	assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+	assert!(net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none());
+
+	// poll until the two nodes connect, otherwise announcing the block will not work
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(0).num_peers() == 0 {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+
+	// synchronization: 0 synced to longer chain and 1 didn't sync to small chain.
+
+	assert_eq!(net.peer(0).client().info().chain.best_number, 40);
+
+	assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+	assert!(!net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+
+	net.peer(0).announce_block(small_hash, Vec::new());
+
+	// after announcing, peer 1 downloads the block.
+
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+
+		assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+		if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() {
+			return Ok(Async::NotReady)
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+	net.block_until_sync(&mut runtime);
+
+	let another_fork = net.peer(0).push_blocks_at(BlockId::Number(35), 2, true);
+	net.peer(0).announce_block(another_fork, Vec::new());
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(1).client().header(&BlockId::Hash(another_fork)).unwrap().is_none() {
+			return Ok(Async::NotReady)
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+}
+
+#[test]
+fn can_not_sync_from_light_peer() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+
+	// given the network with 1 full nodes (#0) and 1 light node (#1)
+	let mut net = TestNet::new(1);
+	net.add_light_peer(&Default::default());
+
+	// generate some blocks on #0
+	net.peer(0).push_blocks(1, false);
+
+	// and let the light client sync from this node
+	net.block_until_sync(&mut runtime);
+
+	// ensure #0 && #1 have the same best block
+	let full0_info = net.peer(0).client.info().chain;
+	let light_info = net.peer(1).client.info().chain;
+	assert_eq!(full0_info.best_number, 1);
+	assert_eq!(light_info.best_number, 1);
+	assert_eq!(light_info.best_hash, full0_info.best_hash);
+
+	// add new full client (#2) && remove #0
+	net.add_full_peer(&Default::default());
+	net.peers.remove(0);
+
+	// ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds
+	let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).compat();
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		test_finished.poll().map_err(|_| ())
+	})).unwrap();
+}
+
+#[test]
+fn light_peer_imports_header_from_announce() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+
+	fn import_with_announce(net: &mut TestNet, runtime: &mut current_thread::Runtime, hash: H256) {
+		net.peer(0).announce_block(hash, Vec::new());
+
+		runtime.block_on(futures::future::poll_fn::<(), (), _>(|| {
+			net.poll();
+			if net.peer(1).client().header(&BlockId::Hash(hash)).unwrap().is_some() {
+				Ok(Async::Ready(()))
+			} else {
+				Ok(Async::NotReady)
+			}
+		})).unwrap();
+	}
+
+	// given the network with 1 full nodes (#0) and 1 light node (#1)
+	let mut net = TestNet::new(1);
+	net.add_light_peer(&Default::default());
+
+	// let them connect to each other
+	net.block_until_sync(&mut runtime);
+
+	// check that NEW block is imported from announce message
+	let new_hash = net.peer(0).push_blocks(1, false);
+	import_with_announce(&mut net, &mut runtime, new_hash);
+
+	// check that KNOWN STALE block is imported from announce message
+	let known_stale_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 1, true);
+	import_with_announce(&mut net, &mut runtime, known_stale_hash);
+}
+
+#[test]
+fn can_sync_explicit_forks() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(2);
+	net.peer(0).push_blocks(30, false);
+	net.peer(1).push_blocks(30, false);
+
+	// small fork + reorg on peer 1.
+	net.peer(0).push_blocks_at(BlockId::Number(30), 2, true);
+	let small_hash = net.peer(0).client().info().chain.best_hash;
+	let small_number = net.peer(0).client().info().chain.best_number;
+	net.peer(0).push_blocks_at(BlockId::Number(30), 10, false);
+	assert_eq!(net.peer(0).client().info().chain.best_number, 40);
+
+	// peer 1 only ever had the long fork.
+	net.peer(1).push_blocks(10, false);
+	assert_eq!(net.peer(1).client().info().chain.best_number, 40);
+
+	assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+	assert!(net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none());
+
+	// poll until the two nodes connect, otherwise announcing the block will not work
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(0).num_peers() == 0  || net.peer(1).num_peers() == 0 {
+			Ok(Async::NotReady)
+		} else {
+			Ok(Async::Ready(()))
+		}
+	})).unwrap();
+
+	// synchronization: 0 synced to longer chain and 1 didn't sync to small chain.
+
+	assert_eq!(net.peer(0).client().info().chain.best_number, 40);
+
+	assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+	assert!(!net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+
+	// request explicit sync
+	let first_peer_id = net.peer(0).id();
+	net.peer(1).set_sync_fork_request(vec![first_peer_id], small_hash, small_number);
+
+	// peer 1 downloads the block.
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+
+		assert!(net.peer(0).client().header(&BlockId::Hash(small_hash)).unwrap().is_some());
+		if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() {
+			return Ok(Async::NotReady)
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+}
+
+#[test]
+fn syncs_header_only_forks() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(0);
+	let config = ProtocolConfig::default();
+	net.add_full_peer_with_states(&config, None);
+	net.add_full_peer_with_states(&config, Some(3));
+	net.peer(0).push_blocks(2, false);
+	net.peer(1).push_blocks(2, false);
+
+	net.peer(0).push_blocks(2, true);
+	let small_hash = net.peer(0).client().info().chain.best_hash;
+	let small_number = net.peer(0).client().info().chain.best_number;
+	net.peer(1).push_blocks(4, false);
+
+	net.block_until_sync(&mut runtime);
+	// Peer 1 will sync the small fork even though common block state is missing
+	assert_eq!(9, net.peer(0).blocks_count());
+	assert_eq!(9, net.peer(1).blocks_count());
+
+	// Request explicit header-only sync request for the ancient fork.
+	let first_peer_id = net.peer(0).id();
+	net.peer(1).set_sync_fork_request(vec![first_peer_id], small_hash, small_number);
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		net.poll();
+		if net.peer(1).client().header(&BlockId::Hash(small_hash)).unwrap().is_none() {
+			return Ok(Async::NotReady)
+		}
+		Ok(Async::Ready(()))
+	})).unwrap();
+}
+
+#[test]
+fn does_not_sync_announced_old_best_block() {
+	let _ = ::env_logger::try_init();
+	let mut runtime = current_thread::Runtime::new().unwrap();
+	let mut net = TestNet::new(3);
+
+	let old_hash = net.peer(0).push_blocks(1, false);
+	let old_hash_with_parent = net.peer(0).push_blocks(1, false);
+	net.peer(0).push_blocks(18, true);
+	net.peer(1).push_blocks(20, true);
+
+	net.peer(0).announce_block(old_hash, Vec::new());
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		// poll once to import announcement
+		net.poll();
+		Ok(Async::Ready(()))
+	})).unwrap();
+	assert!(!net.peer(1).is_major_syncing());
+
+	net.peer(0).announce_block(old_hash_with_parent, Vec::new());
+	runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> {
+		// poll once to import announcement
+		net.poll();
+		Ok(Async::Ready(()))
+	})).unwrap();
+	assert!(!net.peer(1).is_major_syncing());
+}

substrate/client/network/src/transport.rs

@@ -0,0 +1,132 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use futures::prelude::*;
+use libp2p::{
+	InboundUpgradeExt, OutboundUpgradeExt, PeerId, Transport,
+	mplex, identity, secio, yamux, bandwidth, wasm_ext
+};
+#[cfg(not(target_os = "unknown"))]
+use libp2p::{tcp, dns, websocket, noise};
+#[cfg(not(target_os = "unknown"))]
+use libp2p::core::{either::EitherError, either::EitherOutput};
+use libp2p::core::{self, upgrade, transport::boxed::Boxed, transport::OptionalTransport, muxing::StreamMuxerBox};
+use std::{io, sync::Arc, time::Duration, usize};
+
+pub use self::bandwidth::BandwidthSinks;
+
+/// Builds the transport that serves as a common ground for all connections.
+///
+/// If `memory_only` is true, then only communication within the same process are allowed. Only
+/// addresses with the format `/memory/...` are allowed.
+///
+/// Returns a `BandwidthSinks` object that allows querying the average bandwidth produced by all
+/// the connections spawned with this transport.
+pub fn build_transport(
+	keypair: identity::Keypair,
+	memory_only: bool,
+	wasm_external_transport: Option<wasm_ext::ExtTransport>
+) -> (Boxed<(PeerId, StreamMuxerBox), io::Error>, Arc<bandwidth::BandwidthSinks>) {
+	// Build configuration objects for encryption mechanisms.
+	#[cfg(not(target_os = "unknown"))]
+	let noise_config = {
+		let noise_keypair = noise::Keypair::new().into_authentic(&keypair)
+			// For more information about this panic, see in "On the Importance of Checking
+			// Cryptographic Protocols for Faults" by Dan Boneh, Richard A. DeMillo,
+			// and Richard J. Lipton.
+			.expect("can only fail in case of a hardware bug; since this signing is performed only \
+				once and at initialization, we're taking the bet that the inconvenience of a very \
+				rare panic here is basically zero");
+		noise::NoiseConfig::ix(noise_keypair)
+	};
+	let secio_config = secio::SecioConfig::new(keypair);
+
+	// Build configuration objects for multiplexing mechanisms.
+	let mut mplex_config = mplex::MplexConfig::new();
+	mplex_config.max_buffer_len_behaviour(mplex::MaxBufferBehaviour::Block);
+	mplex_config.max_buffer_len(usize::MAX);
+	let yamux_config = yamux::Config::default();
+
+	// Build the base layer of the transport.
+	let transport = if let Some(t) = wasm_external_transport {
+		OptionalTransport::some(t)
+	} else {
+		OptionalTransport::none()
+	};
+	#[cfg(not(target_os = "unknown"))]
+	let transport = transport.or_transport(if !memory_only {
+		let desktop_trans = tcp::TcpConfig::new();
+		let desktop_trans = websocket::WsConfig::new(desktop_trans.clone())
+			.or_transport(desktop_trans);
+		OptionalTransport::some(dns::DnsConfig::new(desktop_trans))
+	} else {
+		OptionalTransport::none()
+	});
+
+	let transport = transport.or_transport(if memory_only {
+		OptionalTransport::some(libp2p::core::transport::MemoryTransport::default())
+	} else {
+		OptionalTransport::none()
+	});
+
+	let (transport, sinks) = bandwidth::BandwidthLogging::new(transport, Duration::from_secs(5));
+
+	// Encryption
+
+	// For non-WASM, we support both secio and noise.
+	#[cfg(not(target_os = "unknown"))]
+	let transport = transport.and_then(move |stream, endpoint| {
+		let upgrade = core::upgrade::SelectUpgrade::new(noise_config, secio_config);
+		core::upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
+			.and_then(|out| match out {
+				// We negotiated noise
+				EitherOutput::First((remote_id, out)) => {
+					let remote_key = match remote_id {
+						noise::RemoteIdentity::IdentityKey(key) => key,
+						_ => return Err(upgrade::UpgradeError::Apply(EitherError::A(noise::NoiseError::InvalidKey)))
+					};
+					Ok((EitherOutput::First(out), remote_key.into_peer_id()))
+				}
+				// We negotiated secio
+				EitherOutput::Second((remote_id, out)) =>
+					Ok((EitherOutput::Second(out), remote_id))
+			})
+	});
+
+	// For WASM, we only support secio for now.
+	#[cfg(target_os = "unknown")]
+	let transport = transport.and_then(move |stream, endpoint| {
+		core::upgrade::apply(stream, secio_config, endpoint, upgrade::Version::V1)
+			.and_then(|(id, stream)| Ok((stream, id)))
+	});
+
+	// Multiplexing
+	let transport = transport.and_then(move |(stream, peer_id), endpoint| {
+			let peer_id2 = peer_id.clone();
+			let upgrade = core::upgrade::SelectUpgrade::new(yamux_config, mplex_config)
+				.map_inbound(move |muxer| (peer_id, muxer))
+				.map_outbound(move |muxer| (peer_id2, muxer));
+
+			core::upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1)
+				.map(|(id, muxer)| (id, core::muxing::StreamMuxerBox::new(muxer)))
+		})
+
+		.timeout(Duration::from_secs(20))
+		.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+		.boxed();
+
+	(transport, sinks)
+}