Network crate cleanups (#3049)

* Remove useless internal messages

* Remove NetworkService::disconnect_peer

* Remove NetworkMsg altogether

* Rename ProtocolMsg ServerToWorkerMsg

* Remove useless code

* Add example for parse_str_addr

* Move parse_str_addr and ProtocolId to config

* Don't reexport the content of config

* Rework the imports

* More reexports rework

* Add documentation

* Move finalization report to network future

* Move on_block_imported to worker

* get_value/put_value no longer locking

* local_peer_id() no longer locks

* Remove FetchFuture

* Service imports cleanup

* Produce the network state in the network task

* Merge network task and RPC network task

* Move network methods to NetworkWorker

* Remove Arc peers system from network

* add_reserved_peer now goes through the channel

* Remove Mutex around network swarm

* Remove the FnOnce alias traits

* Replace is_offline with num_connected

* Improve style of poll()

* Fix network tests

* Some doc in service module

* Remove macro export

* Minor doc changes

* Remove the synchronized() method of the import queue

* Line width

* Line widths

* Fix import queue tests

* Fix CLI tests

substrate/core/network/src/test/mod.rs

@@ -24,10 +24,10 @@ mod sync;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use crate::build_multiaddr;
+use crate::config::build_multiaddr;
 use log::trace;
 use crate::chain::FinalityProofProvider;
-use client::{self, ClientInfo, BlockchainEvents, FinalityNotifications};
+use client::{self, ClientInfo, BlockchainEvents, ImportNotifications, FinalityNotifications};
 use client::{in_mem::Backend as InMemoryBackend, error::Result as ClientResult};
 use client::block_builder::BlockBuilder;
 use client::backend::AuxStore;
@@ -41,7 +41,7 @@ use consensus::block_import::BlockImport;
 use consensus::{Error as ConsensusError, well_known_cache_keys::{self, Id as CacheKeyId}};
 use consensus::{BlockOrigin, ForkChoiceStrategy, ImportBlock, JustificationImport};
 use futures::prelude::*;
-use crate::{NetworkWorker, NetworkService, ProtocolId};
+use crate::{NetworkWorker, NetworkService, config::ProtocolId};
 use crate::config::{NetworkConfiguration, TransportConfig};
 use libp2p::PeerId;
 use primitives::{H256, Blake2Hasher};
@@ -119,7 +119,7 @@ impl NetworkSpecialization<Block> for DummySpecialization {
 
 	fn on_event(
 		&mut self,
-		_event: crate::event::Event
+		_event: crate::specialization::Event
 	) {}
 }
 
@@ -216,7 +216,8 @@ pub struct Peer<D, S: NetworkSpecialization<Block>> {
 	/// instead of going through the import queue.
 	block_import: Arc<dyn BlockImport<Block, Error = ConsensusError>>,
 	network: NetworkWorker<Block, S, <Block as BlockT>::Hash>,
-	to_poll: smallvec::SmallVec<[Box<dyn Future<Item = (), Error = ()> + Send>; 2]>,
+	imported_blocks_stream: futures::stream::Fuse<ImportNotifications<Block>>,
+	finality_notification_stream: futures::stream::Fuse<FinalityNotifications<Block>>,
 }
 
 impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
@@ -246,7 +247,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 	}
 
 	/// Add blocks to the peer -- edit the block before adding
-	pub fn generate_blocks<F>(&self, count: usize, origin: BlockOrigin, edit_block: F) -> H256
+	pub fn generate_blocks<F>(&mut self, count: usize, origin: BlockOrigin, edit_block: F) -> H256
 		where F: FnMut(BlockBuilder<Block, PeersFullClient>) -> Block
 	{
 		let best_hash = self.client.info().chain.best_hash;
@@ -256,7 +257,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 	/// Add blocks to the peer -- edit the block before adding. The chain will
 	/// start at the given block iD.
 	fn generate_blocks_at<F>(
-		&self,
+		&mut self,
 		at: BlockId<Block>,
 		count: usize,
 		origin: BlockOrigin,
@@ -289,7 +290,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 				Default::default()
 			};
 			self.block_import.import_block(import_block, cache).expect("block_import failed");
-			self.network.service().on_block_imported(hash, header);
+			self.network.on_block_imported(hash, header);
 			at = hash;
 		}
 
@@ -298,14 +299,14 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 	}
 
 	/// Push blocks to the peer (simplified: with or without a TX)
-	pub fn push_blocks(&self, count: usize, with_tx: bool) -> H256 {
+	pub fn push_blocks(&mut self, count: usize, with_tx: bool) -> H256 {
 		let best_hash = self.client.info().chain.best_hash;
 		self.push_blocks_at(BlockId::Hash(best_hash), count, with_tx)
 	}
 
 	/// Push blocks to the peer (simplified: with or without a TX) starting from
 	/// given hash.
-	pub fn push_blocks_at(&self, at: BlockId<Block>, count: usize, with_tx: bool) -> H256 {
+	pub fn push_blocks_at(&mut self, at: BlockId<Block>, count: usize, with_tx: bool) -> H256 {
 		let mut nonce = 0;
 		if with_tx {
 			self.generate_blocks_at(at, count, BlockOrigin::File, |mut builder| {
@@ -324,7 +325,7 @@ impl<D, S: NetworkSpecialization<Block>> Peer<D, S> {
 		}
 	}
 
-	pub fn push_authorities_change_block(&self, new_authorities: Vec<AuthorityId>) -> H256 {
+	pub fn push_authorities_change_block(&mut self, new_authorities: Vec<AuthorityId>) -> H256 {
 		self.generate_blocks(1, BlockOrigin::File, |mut builder| {
 			builder.push(Extrinsic::AuthoritiesChange(new_authorities.clone())).unwrap();
 			builder.bake().unwrap()
@@ -376,7 +377,7 @@ pub trait TestNetFactory: Sized {
 	fn make_verifier(&self, client: PeersClient, config: &ProtocolConfig) -> Arc<Self::Verifier>;
 
 	/// Get reference to peer.
-	fn peer(&self, i: usize) -> &Peer<Self::PeerData, Self::Specialization>;
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::Specialization>;
 	fn peers(&self) -> &Vec<Peer<Self::PeerData, Self::Specialization>>;
 	fn mut_peers<F: FnOnce(&mut Vec<Peer<Self::PeerData, Self::Specialization>>)>(&mut self, closure: F);
 
@@ -448,76 +449,21 @@ pub trait TestNetFactory: Sized {
 			specialization: self::SpecializationFactory::create(),
 		}).unwrap();
 
-		let blocks_notif_future = {
-			let network = Arc::downgrade(&network.service().clone());
-			client.import_notification_stream()
-				.for_each(move |notification| {
-					if let Some(network) = network.upgrade() {
-						network.on_block_imported(notification.hash, notification.header);
-					}
-					Ok(())
-				})
-				.then(|_| Ok(()))
-		};
-
-		let finality_notif_future = {
-			let network = Arc::downgrade(&network.service().clone());
-
-			// A utility stream that drops all ready items and only returns the last one.
-			// This is used to only keep the last finality notification and avoid
-			// overloading the sync module with notifications.
-			struct MostRecentNotification<B: BlockT>(futures::stream::Fuse<FinalityNotifications<B>>);
-
-			impl<B: BlockT> Stream for MostRecentNotification<B> {
-				type Item = <FinalityNotifications<B> as Stream>::Item;
-				type Error = <FinalityNotifications<B> as Stream>::Error;
-
-				fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
-					let mut last = None;
-					let last = loop {
-						match self.0.poll()? {
-							Async::Ready(Some(item)) => { last = Some(item) }
-							Async::Ready(None) => match last {
-								None => return Ok(Async::Ready(None)),
-								Some(last) => break last,
-							},
-							Async::NotReady => match last {
-								None => return Ok(Async::NotReady),
-								Some(last) => break last,
-							},
-						}
-					};
-
-					Ok(Async::Ready(Some(last)))
-				}
-			}
-
-			MostRecentNotification(client.finality_notification_stream().fuse())
-				.for_each(move |notification| {
-					if let Some(network) = network.upgrade() {
-						network.on_block_finalized(notification.hash, notification.header);
-					}
-					Ok(())
-				})
-				.then(|_| Ok(()))
-		};
-
 		self.mut_peers(|peers| {
 			for peer in peers.iter_mut() {
 				peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone());
 			}
 
+			let imported_blocks_stream = client.import_notification_stream().fuse();
+			let finality_notification_stream = client.finality_notification_stream().fuse();
+
 			peers.push(Peer {
 				data,
 				client: PeersClient::Full(client),
+				imported_blocks_stream,
+				finality_notification_stream,
 				block_import,
 				verifier,
-				to_poll: {
-					let mut sv = smallvec::SmallVec::new();
-					sv.push(Box::new(blocks_notif_future) as Box<_>);
-					sv.push(Box::new(finality_notif_future) as Box<_>);
-					sv
-				},
 				network,
 			});
 		});
@@ -559,33 +505,21 @@ pub trait TestNetFactory: Sized {
 			specialization: self::SpecializationFactory::create(),
 		}).unwrap();
 
-		let blocks_notif_future = {
-			let network = Arc::downgrade(&network.service().clone());
-			client.import_notification_stream()
-				.for_each(move |notification| {
-					if let Some(network) = network.upgrade() {
-						network.on_block_imported(notification.hash, notification.header);
-					}
-					Ok(())
-				})
-				.then(|_| Ok(()))
-		};
-
 		self.mut_peers(|peers| {
 			for peer in peers.iter_mut() {
 				peer.network.add_known_address(network.service().local_peer_id(), listen_addr.clone());
 			}
 
+			let imported_blocks_stream = client.import_notification_stream().fuse();
+			let finality_notification_stream = client.finality_notification_stream().fuse();
+
 			peers.push(Peer {
 				data,
 				verifier,
 				block_import,
 				client: PeersClient::Light(client),
-				to_poll: {
-					let mut sv = smallvec::SmallVec::new();
-					sv.push(Box::new(blocks_notif_future) as Box<_>);
-					sv
-				},
+				imported_blocks_stream,
+				finality_notification_stream,
 				network,
 			});
 		});
@@ -621,7 +555,20 @@ pub trait TestNetFactory: Sized {
 		self.mut_peers(|peers| {
 			for peer in peers {
 				peer.network.poll().unwrap();
-				peer.to_poll.retain(|f| f.poll() == Ok(Async::NotReady));
+
+				// We poll `imported_blocks_stream`.
+				while let Ok(Async::Ready(Some(notification))) = peer.imported_blocks_stream.poll() {
+					peer.network.on_block_imported(notification.hash, notification.header);
+				}
+
+				// We poll `finality_notification_stream`, but we only take the last event.
+				let mut last = None;
+				while let Ok(Async::Ready(Some(item))) = peer.finality_notification_stream.poll() {
+					last = Some(item);
+				}
+				if let Some(notification) = last {
+					peer.network.on_block_finalized(notification.hash, notification.header);
+				}
 			}
 		});
 	}
@@ -649,8 +596,8 @@ impl TestNetFactory for TestNet {
 		Arc::new(PassThroughVerifier(false))
 	}
 
-	fn peer(&self, i: usize) -> &Peer<(), Self::Specialization> {
-		&self.peers[i]
+	fn peer(&mut self, i: usize) -> &mut Peer<(), Self::Specialization> {
+		&mut self.peers[i]
 	}
 
 	fn peers(&self) -> &Vec<Peer<(), Self::Specialization>> {
@@ -695,7 +642,7 @@ impl TestNetFactory for JustificationTestNet {
 		self.0.make_verifier(client, config)
 	}
 
-	fn peer(&self, i: usize) -> &Peer<Self::PeerData, Self::Specialization> {
+	fn peer(&mut self, i: usize) -> &mut Peer<Self::PeerData, Self::Specialization> {
 		self.0.peer(i)
 	}