Extract syncing protocol from sc-network (#12828)

* Move import queue out of `sc-network`

Add supplementary asynchronous API for the import queue which means
it can be run as an independent task and communicated with through
the `ImportQueueService`.

This commit removes removes block and justification imports from
`sc-network` and provides `ChainSync` with a handle to import queue so
it can import blocks and justifications. Polling of the import queue is
moved complete out of `sc-network` and `sc_consensus::Link` is
implemented for `ChainSyncInterfaceHandled` so the import queue
can still influence the syncing process.

* Move stuff to SyncingEngine

* Move `ChainSync` instanation to `SyncingEngine`

Some of the tests have to be rewritten

* Move peer hashmap to `SyncingEngine`

* Let `SyncingEngine` to implement `ChainSyncInterface`

* Introduce `SyncStatusProvider`

* Move `sync_peer_(connected|disconnected)` to `SyncingEngine`

* Implement `SyncEventStream`

Remove `SyncConnected`/`SyncDisconnected` events from
`NetworkEvenStream` and provide those events through
`ChainSyncInterface` instead.

Modify BEEFY/GRANDPA/transactions protocol and `NetworkGossip` to take
`SyncEventStream` object which they listen to for incoming sync peer
events.

* Introduce `ChainSyncInterface`

This interface provides a set of miscellaneous functions that other
subsystems can use to query, for example, the syncing status.

* Move event stream polling to `SyncingEngine`

Subscribe to `NetworkStreamEvent` and poll the incoming notifications
and substream events from `SyncingEngine`.

The code needs refactoring.

* Make `SyncingEngine` into an asynchronous runner

This commits removes the last hard dependency of syncing from
`sc-network` meaning the protocol now lives completely outside of
`sc-network`, ignoring the hardcoded peerset entry which will be
addressed in the future.

Code needs a lot of refactoring.

* Fix warnings

* Code refactoring

* Use `SyncingService` for BEEFY

* Use `SyncingService` for GRANDPA

* Remove call delegation from `NetworkService`

* Remove `ChainSyncService`

* Remove `ChainSync` service tests

They were written for the sole purpose of verifying that `NetworWorker`
continues to function while the calls are being dispatched to
`ChainSync`.

* Refactor code

* Refactor code

* Update client/finality-grandpa/src/communication/tests.rs

Co-authored-by: Anton <anton.kalyaev@gmail.com>

* Fix warnings

* Apply review comments

* Fix docs

* Fix test

* cargo-fmt

* Update client/network/sync/src/engine.rs

Co-authored-by: Anton <anton.kalyaev@gmail.com>

* Update client/network/sync/src/engine.rs

Co-authored-by: Anton <anton.kalyaev@gmail.com>

* Add missing docs

* Refactor code

---------

Co-authored-by: Anton <anton.kalyaev@gmail.com>

substrate/client/network-gossip/src/bridge.rs

@@ -18,10 +18,13 @@
 
 use crate::{
 	state_machine::{ConsensusGossip, TopicNotification, PERIODIC_MAINTENANCE_INTERVAL},
-	Network, Validator,
+	Network, Syncing, Validator,
 };
 
-use sc_network_common::protocol::{event::Event, ProtocolName};
+use sc_network_common::{
+	protocol::{event::Event, ProtocolName},
+	sync::SyncEvent,
+};
 use sc_peerset::ReputationChange;
 
 use futures::{
@@ -44,11 +47,14 @@ use std::{
 pub struct GossipEngine<B: BlockT> {
 	state_machine: ConsensusGossip<B>,
 	network: Box<dyn Network<B> + Send>,
+	sync: Box<dyn Syncing<B>>,
 	periodic_maintenance_interval: futures_timer::Delay,
 	protocol: ProtocolName,
 
 	/// Incoming events from the network.
 	network_event_stream: Pin<Box<dyn Stream<Item = Event> + Send>>,
+	/// Incoming events from the syncing service.
+	sync_event_stream: Pin<Box<dyn Stream<Item = SyncEvent> + Send>>,
 	/// Outgoing events to the consumer.
 	message_sinks: HashMap<B::Hash, Vec<Sender<TopicNotification>>>,
 	/// Buffered messages (see [`ForwardingState`]).
@@ -75,25 +81,31 @@ impl<B: BlockT> Unpin for GossipEngine<B> {}
 
 impl<B: BlockT> GossipEngine<B> {
 	/// Create a new instance.
-	pub fn new<N: Network<B> + Send + Clone + 'static>(
+	pub fn new<N, S>(
 		network: N,
+		sync: S,
 		protocol: impl Into<ProtocolName>,
 		validator: Arc<dyn Validator<B>>,
 		metrics_registry: Option<&Registry>,
 	) -> Self
 	where
 		B: 'static,
+		N: Network<B> + Send + Clone + 'static,
+		S: Syncing<B> + Send + Clone + 'static,
 	{
 		let protocol = protocol.into();
 		let network_event_stream = network.event_stream("network-gossip");
+		let sync_event_stream = sync.event_stream("network-gossip");
 
 		GossipEngine {
 			state_machine: ConsensusGossip::new(validator, protocol.clone(), metrics_registry),
 			network: Box::new(network),
+			sync: Box::new(sync),
 			periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL),
 			protocol,
 
 			network_event_stream,
+			sync_event_stream,
 			message_sinks: HashMap::new(),
 			forwarding_state: ForwardingState::Idle,
 
@@ -162,7 +174,7 @@ impl<B: BlockT> GossipEngine<B> {
 	/// Note: this method isn't strictly related to gossiping and should eventually be moved
 	/// somewhere else.
 	pub fn announce(&self, block: B::Hash, associated_data: Option<Vec<u8>>) {
-		self.network.announce_block(block, associated_data);
+		self.sync.announce_block(block, associated_data);
 	}
 }
 
@@ -175,28 +187,24 @@ impl<B: BlockT> Future for GossipEngine<B> {
 		'outer: loop {
 			match &mut this.forwarding_state {
 				ForwardingState::Idle => {
-					match this.network_event_stream.poll_next_unpin(cx) {
+					let net_event_stream = this.network_event_stream.poll_next_unpin(cx);
+					let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx);
+
+					if net_event_stream.is_pending() && sync_event_stream.is_pending() {
+						break
+					}
+
+					match net_event_stream {
 						Poll::Ready(Some(event)) => match event {
-							Event::SyncConnected { remote } => {
-								this.network.add_set_reserved(remote, this.protocol.clone());
-							},
-							Event::SyncDisconnected { remote } => {
-								this.network.remove_peers_from_reserved_set(
-									this.protocol.clone(),
-									vec![remote],
-								);
-							},
-							Event::NotificationStreamOpened { remote, protocol, role, .. } => {
-								if protocol != this.protocol {
-									continue
-								}
-								this.state_machine.new_peer(&mut *this.network, remote, role);
-							},
+							Event::NotificationStreamOpened { remote, protocol, role, .. } =>
+								if protocol == this.protocol {
+									this.state_machine.new_peer(&mut *this.network, remote, role);
+								},
 							Event::NotificationStreamClosed { remote, protocol } => {
-								if protocol != this.protocol {
-									continue
+								if protocol == this.protocol {
+									this.state_machine
+										.peer_disconnected(&mut *this.network, remote);
 								}
-								this.state_machine.peer_disconnected(&mut *this.network, remote);
 							},
 							Event::NotificationsReceived { remote, messages } => {
 								let messages = messages
@@ -225,7 +233,25 @@ impl<B: BlockT> Future for GossipEngine<B> {
 							self.is_terminated = true;
 							return Poll::Ready(())
 						},
-						Poll::Pending => break,
+						Poll::Pending => {},
+					}
+
+					match sync_event_stream {
+						Poll::Ready(Some(event)) => match event {
+							SyncEvent::PeerConnected(remote) =>
+								this.network.add_set_reserved(remote, this.protocol.clone()),
+							SyncEvent::PeerDisconnected(remote) =>
+								this.network.remove_peers_from_reserved_set(
+									this.protocol.clone(),
+									vec![remote],
+								),
+						},
+						// The sync event stream closed. Do the same for [`GossipValidator`].
+						Poll::Ready(None) => {
+							self.is_terminated = true;
+							return Poll::Ready(())
+						},
+						Poll::Pending => {},
 					}
 				},
 				ForwardingState::Busy(to_forward) => {
@@ -321,6 +347,7 @@ mod tests {
 			NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers,
 			NotificationSender, NotificationSenderError,
 		},
+		sync::SyncEventStream,
 	};
 	use sp_runtime::{
 		testing::H256,
@@ -433,6 +460,10 @@ mod tests {
 		) -> Result<Box<dyn NotificationSender>, NotificationSenderError> {
 			unimplemented!();
 		}
+
+		fn set_notification_handshake(&self, _protocol: ProtocolName, _handshake: Vec<u8>) {
+			unimplemented!();
+		}
 	}
 
 	impl NetworkBlock<<Block as BlockT>::Hash, NumberFor<Block>> for TestNetwork {
@@ -449,6 +480,42 @@ mod tests {
 		}
 	}
 
+	#[derive(Clone, Default)]
+	struct TestSync {
+		inner: Arc<Mutex<TestSyncInner>>,
+	}
+
+	#[derive(Clone, Default)]
+	struct TestSyncInner {
+		event_senders: Vec<UnboundedSender<SyncEvent>>,
+	}
+
+	impl SyncEventStream for TestSync {
+		fn event_stream(
+			&self,
+			_name: &'static str,
+		) -> Pin<Box<dyn Stream<Item = SyncEvent> + Send>> {
+			let (tx, rx) = unbounded();
+			self.inner.lock().unwrap().event_senders.push(tx);
+
+			Box::pin(rx)
+		}
+	}
+
+	impl NetworkBlock<<Block as BlockT>::Hash, NumberFor<Block>> for TestSync {
+		fn announce_block(&self, _hash: <Block as BlockT>::Hash, _data: Option<Vec<u8>>) {
+			unimplemented!();
+		}
+
+		fn new_best_block_imported(
+			&self,
+			_hash: <Block as BlockT>::Hash,
+			_number: NumberFor<Block>,
+		) {
+			unimplemented!();
+		}
+	}
+
 	struct AllowAll;
 	impl Validator<Block> for AllowAll {
 		fn validate(
@@ -468,8 +535,10 @@ mod tests {
 	#[test]
 	fn returns_when_network_event_stream_closes() {
 		let network = TestNetwork::default();
+		let sync = Arc::new(TestSync::default());
 		let mut gossip_engine = GossipEngine::<Block>::new(
 			network.clone(),
+			sync,
 			"/my_protocol",
 			Arc::new(AllowAll {}),
 			None,
@@ -495,9 +564,11 @@ mod tests {
 		let protocol = ProtocolName::from("/my_protocol");
 		let remote_peer = PeerId::random();
 		let network = TestNetwork::default();
+		let sync = Arc::new(TestSync::default());
 
 		let mut gossip_engine = GossipEngine::<Block>::new(
 			network.clone(),
+			sync.clone(),
 			protocol.clone(),
 			Arc::new(AllowAll {}),
 			None,
@@ -512,6 +583,7 @@ mod tests {
 				protocol: protocol.clone(),
 				negotiated_fallback: None,
 				role: ObservedRole::Authority,
+				received_handshake: vec![],
 			})
 			.expect("Event stream is unbounded; qed.");
 
@@ -614,6 +686,7 @@ mod tests {
 			let protocol = ProtocolName::from("/my_protocol");
 			let remote_peer = PeerId::random();
 			let network = TestNetwork::default();
+			let sync = Arc::new(TestSync::default());
 
 			let num_channels_per_topic = channels.iter().fold(
 				HashMap::new(),
@@ -640,6 +713,7 @@ mod tests {
 
 			let mut gossip_engine = GossipEngine::<Block>::new(
 				network.clone(),
+				sync.clone(),
 				protocol.clone(),
 				Arc::new(TestValidator {}),
 				None,
@@ -674,6 +748,7 @@ mod tests {
 					protocol: protocol.clone(),
 					negotiated_fallback: None,
 					role: ObservedRole::Authority,
+					received_handshake: vec![],
 				})
 				.expect("Event stream is unbounded; qed.");
 

substrate/client/network-gossip/src/lib.rs

@@ -71,6 +71,7 @@ use libp2p::{multiaddr, PeerId};
 use sc_network_common::{
 	protocol::ProtocolName,
 	service::{NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers},
+	sync::SyncEventStream,
 };
 use sp_runtime::traits::{Block as BlockT, NumberFor};
 use std::iter;
@@ -80,9 +81,7 @@ mod state_machine;
 mod validator;
 
 /// Abstraction over a network.
-pub trait Network<B: BlockT>:
-	NetworkPeers + NetworkEventStream + NetworkNotification + NetworkBlock<B::Hash, NumberFor<B>>
-{
+pub trait Network<B: BlockT>: NetworkPeers + NetworkEventStream + NetworkNotification {
 	fn add_set_reserved(&self, who: PeerId, protocol: ProtocolName) {
 		let addr =
 			iter::once(multiaddr::Protocol::P2p(who.into())).collect::<multiaddr::Multiaddr>();
@@ -93,10 +92,9 @@ pub trait Network<B: BlockT>:
 	}
 }
 
-impl<T, B: BlockT> Network<B> for T where
-	T: NetworkPeers
-		+ NetworkEventStream
-		+ NetworkNotification
-		+ NetworkBlock<B::Hash, NumberFor<B>>
-{
-}
+impl<T, B: BlockT> Network<B> for T where T: NetworkPeers + NetworkEventStream + NetworkNotification {}
+
+/// Abstraction over the syncing subsystem.
+pub trait Syncing<B: BlockT>: SyncEventStream + NetworkBlock<B::Hash, NumberFor<B>> {}
+
+impl<T, B: BlockT> Syncing<B> for T where T: SyncEventStream + NetworkBlock<B::Hash, NumberFor<B>> {}

substrate/client/network-gossip/src/state_machine.rs

@@ -683,6 +683,10 @@ mod tests {
 		) -> Result<Box<dyn NotificationSender>, NotificationSenderError> {
 			unimplemented!();
 		}
+
+		fn set_notification_handshake(&self, _protocol: ProtocolName, _handshake: Vec<u8>) {
+			unimplemented!();
+		}
 	}
 
 	impl NetworkBlock<<Block as BlockT>::Hash, NumberFor<Block>> for NoOpNetwork {