Rework the event system of sc-network (#1370)

This commit introduces a new concept called `NotificationService` which
allows Polkadot protocols to communicate with the underlying
notification protocol implementation directly, without routing events
through `NetworkWorker`. This implies that each protocol has its own
service which it uses to communicate with remote peers and that each
`NotificationService` is unique with respect to the underlying
notification protocol, meaning `NotificationService` for the transaction
protocol can only be used to send and receive transaction-related
notifications.

The `NotificationService` concept introduces two additional benefits:
  * allow protocols to start using custom handshakes
  * allow protocols to accept/reject inbound peers

Previously the validation of inbound connections was solely the
responsibility of `ProtocolController`. This caused issues with light
peers and `SyncingEngine` as `ProtocolController` would accept more
peers than `SyncingEngine` could accept which caused peers to have
differing views of their own states. `SyncingEngine` would reject excess
peers but these rejections were not properly communicated to those peers
causing them to assume that they were accepted.

With `NotificationService`, the local handshake is not sent to remote
peer if peer is rejected which allows it to detect that it was rejected.

This commit also deprecates the use of `NetworkEventStream` for all
notification-related events and going forward only DHT events are
provided through `NetworkEventStream`. If protocols wish to follow each
other's events, they must introduce additional abtractions, as is done
for GRANDPA and transactions protocols by following the syncing protocol
through `SyncEventStream`.

Fixes https://github.com/paritytech/polkadot-sdk/issues/512
Fixes https://github.com/paritytech/polkadot-sdk/issues/514
Fixes https://github.com/paritytech/polkadot-sdk/issues/515
Fixes https://github.com/paritytech/polkadot-sdk/issues/554
Fixes https://github.com/paritytech/polkadot-sdk/issues/556

---
These changes are transferred from
https://github.com/paritytech/substrate/pull/14197 but there are no
functional changes compared to that PR

---------

Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

substrate/client/consensus/beefy/src/communication/mod.rs

@@ -67,10 +67,16 @@ pub(crate) mod beefy_protocol_name {
 /// For standard protocol name see [`beefy_protocol_name::gossip_protocol_name`].
 pub fn beefy_peers_set_config(
 	gossip_protocol_name: sc_network::ProtocolName,
-) -> sc_network::config::NonDefaultSetConfig {
-	let mut cfg = sc_network::config::NonDefaultSetConfig::new(gossip_protocol_name, 1024 * 1024);
+) -> (sc_network::config::NonDefaultSetConfig, Box<dyn sc_network::NotificationService>) {
+	let (mut cfg, notification_service) = sc_network::config::NonDefaultSetConfig::new(
+		gossip_protocol_name,
+		Vec::new(),
+		1024 * 1024,
+		None,
+		Default::default(),
+	);
 	cfg.allow_non_reserved(25, 25);
-	cfg
+	(cfg, notification_service)
 }
 
 // cost scalars for reporting peers.

substrate/client/consensus/beefy/src/lib.rs

@@ -38,7 +38,7 @@ use parking_lot::Mutex;
 use prometheus::Registry;
 use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer};
 use sc_consensus::BlockImport;
-use sc_network::{NetworkRequest, ProtocolName};
+use sc_network::{NetworkRequest, NotificationService, ProtocolName};
 use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::{
@@ -178,6 +178,8 @@ pub struct BeefyNetworkParams<B: Block, N, S> {
 	pub network: Arc<N>,
 	/// Syncing service implementing a sync oracle and an event stream for peers.
 	pub sync: Arc<S>,
+	/// Handle for receiving notification events.
+	pub notification_service: Box<dyn NotificationService>,
 	/// Chain specific BEEFY gossip protocol name. See
 	/// [`communication::beefy_protocol_name::gossip_protocol_name`].
 	pub gossip_protocol_name: ProtocolName,
@@ -243,6 +245,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	let BeefyNetworkParams {
 		network,
 		sync,
+		notification_service,
 		gossip_protocol_name,
 		justifications_protocol_name,
 		..
@@ -264,6 +267,7 @@ pub async fn start_beefy_gadget<B, BE, C, N, P, R, S>(
 	let gossip_engine = GossipEngine::new(
 		network.clone(),
 		sync.clone(),
+		notification_service,
 		gossip_protocol_name.clone(),
 		gossip_validator.clone(),
 		None,

substrate/client/consensus/beefy/src/tests.rs

@@ -72,7 +72,7 @@ use substrate_test_runtime_client::{BlockBuilderExt, ClientExt};
 use tokio::time::Duration;
 
 const GENESIS_HASH: H256 = H256::zero();
-fn beefy_gossip_proto_name() -> ProtocolName {
+pub(crate) fn beefy_gossip_proto_name() -> ProtocolName {
 	gossip_protocol_name(GENESIS_HASH, None)
 }
 
@@ -371,6 +371,7 @@ async fn voter_init_setup(
 	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
 		net.peer(0).network_service().clone(),
 		net.peer(0).sync_service().clone(),
+		net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(),
 		"/beefy/whatever",
 		gossip_validator,
 		None,
@@ -392,6 +393,14 @@ where
 {
 	let tasks = FuturesUnordered::new();
 
+	let mut notification_services = peers
+		.iter()
+		.map(|(peer_id, _, _)| {
+			let peer = &mut net.peers[*peer_id];
+			(*peer_id, peer.take_notification_service(&beefy_gossip_proto_name()).unwrap())
+		})
+		.collect::<std::collections::HashMap<_, _>>();
+
 	for (peer_id, key, api) in peers.into_iter() {
 		let peer = &net.peers[peer_id];
 
@@ -409,6 +418,7 @@ where
 		let network_params = crate::BeefyNetworkParams {
 			network: peer.network_service().clone(),
 			sync: peer.sync_service().clone(),
+			notification_service: notification_services.remove(&peer_id).unwrap(),
 			gossip_protocol_name: beefy_gossip_proto_name(),
 			justifications_protocol_name: on_demand_justif_handler.protocol_name(),
 			_phantom: PhantomData,
@@ -1045,7 +1055,25 @@ async fn should_initialize_voter_at_custom_genesis() {
 	net.peer(0).client().as_client().finalize_block(hashes[8], None).unwrap();
 
 	// load persistent state - nothing in DB, should init at genesis
-	let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	//
+	// NOTE: code from `voter_init_setup()` is moved here because the new network event system
+	// doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the
+	// first `GossipEngine`
+	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
+	let (gossip_validator, _) = GossipValidator::new(known_peers);
+	let gossip_validator = Arc::new(gossip_validator);
+	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
+		net.peer(0).network_service().clone(),
+		net.peer(0).sync_service().clone(),
+		net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(),
+		"/beefy/whatever",
+		gossip_validator,
+		None,
+	);
+	let (beefy_genesis, best_grandpa) =
+		wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap();
+	let persisted_state =
+		load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap();
 
 	// Test initialization at session boundary.
 	// verify voter initialized with single session starting at block `custom_pallet_genesis` (7)
@@ -1075,7 +1103,11 @@ async fn should_initialize_voter_at_custom_genesis() {
 
 	net.peer(0).client().as_client().finalize_block(hashes[10], None).unwrap();
 	// load persistent state - state preset in DB, but with different pallet genesis
-	let new_persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap();
+	// the network state persists and uses the old `GossipEngine` initialized for `peer(0)`
+	let (beefy_genesis, best_grandpa) =
+		wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap();
+	let new_persisted_state =
+		load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap();
 
 	// verify voter initialized with single session starting at block `new_pallet_genesis` (10)
 	let sessions = new_persisted_state.voting_oracle().sessions();
@@ -1371,7 +1403,7 @@ async fn gossipped_finality_proofs() {
 	let api = Arc::new(TestApi::with_validator_set(&validator_set));
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 
-	let charlie = &net.peers[2];
+	let charlie = &mut net.peers[2];
 	let known_peers = Arc::new(Mutex::new(KnownPeers::<Block>::new()));
 	// Charlie will run just the gossip engine and not the full voter.
 	let (gossip_validator, _) = GossipValidator::new(known_peers);
@@ -1384,6 +1416,7 @@ async fn gossipped_finality_proofs() {
 	let mut charlie_gossip_engine = sc_network_gossip::GossipEngine::new(
 		charlie.network_service().clone(),
 		charlie.sync_service().clone(),
+		charlie.take_notification_service(&beefy_gossip_proto_name()).unwrap(),
 		beefy_gossip_proto_name(),
 		charlie_gossip_validator.clone(),
 		None,

substrate/client/consensus/beefy/src/worker.rs

@@ -1145,12 +1145,16 @@ pub(crate) mod tests {
 		let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set));
 		let network = peer.network_service().clone();
 		let sync = peer.sync_service().clone();
+		let notification_service = peer
+			.take_notification_service(&crate::tests::beefy_gossip_proto_name())
+			.unwrap();
 		let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 		let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone());
 		let gossip_validator = Arc::new(gossip_validator);
 		let gossip_engine = GossipEngine::new(
 			network.clone(),
 			sync.clone(),
+			notification_service,
 			"/beefy/1",
 			gossip_validator.clone(),
 			None,

substrate/client/consensus/grandpa/src/communication/mod.rs

@@ -46,7 +46,7 @@ use finality_grandpa::{
 	Message::{Precommit, Prevote, PrimaryPropose},
 };
 use parity_scale_codec::{Decode, DecodeAll, Encode};
-use sc_network::{NetworkBlock, NetworkSyncForkRequest, ReputationChange};
+use sc_network::{NetworkBlock, NetworkSyncForkRequest, NotificationService, ReputationChange};
 use sc_network_gossip::{GossipEngine, Network as GossipNetwork};
 use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO};
 use sp_keystore::KeystorePtr;
@@ -247,6 +247,7 @@ impl<B: BlockT, N: Network<B>, S: Syncing<B>> NetworkBridge<B, N, S> {
 	pub(crate) fn new(
 		service: N,
 		sync: S,
+		notification_service: Box<dyn NotificationService>,
 		config: crate::Config,
 		set_state: crate::environment::SharedVoterSetState<B>,
 		prometheus_registry: Option<&Registry>,
@@ -260,6 +261,7 @@ impl<B: BlockT, N: Network<B>, S: Syncing<B>> NetworkBridge<B, N, S> {
 		let gossip_engine = Arc::new(Mutex::new(GossipEngine::new(
 			service.clone(),
 			sync.clone(),
+			notification_service,
 			protocol,
 			validator.clone(),
 			prometheus_registry,

substrate/client/consensus/grandpa/src/communication/tests.rs

@@ -24,16 +24,17 @@ use super::{
 };
 use crate::{communication::grandpa_protocol_name, environment::SharedVoterSetState};
 use futures::prelude::*;
-use parity_scale_codec::Encode;
+use parity_scale_codec::{DecodeAll, Encode};
 use sc_network::{
 	config::{MultiaddrWithPeerId, Role},
 	event::Event as NetworkEvent,
+	service::traits::{Direction, MessageSink, NotificationEvent, NotificationService},
 	types::ProtocolName,
 	Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers,
 	NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender,
 	PeerId, ReputationChange,
 };
-use sc_network_common::role::ObservedRole;
+use sc_network_common::role::{ObservedRole, Roles};
 use sc_network_gossip::Validator;
 use sc_network_sync::{SyncEvent as SyncStreamEvent, SyncEventStream};
 use sc_network_test::{Block, Hash};
@@ -123,6 +124,12 @@ impl NetworkPeers for TestNetwork {
 	fn sync_num_connected(&self) -> usize {
 		unimplemented!();
 	}
+
+	fn peer_role(&self, _peer_id: PeerId, handshake: Vec<u8>) -> Option<ObservedRole> {
+		Roles::decode_all(&mut &handshake[..])
+			.ok()
+			.and_then(|role| Some(ObservedRole::from(role)))
+	}
 }
 
 impl NetworkEventStream for TestNetwork {
@@ -211,10 +218,70 @@ impl NetworkSyncForkRequest<Hash, NumberFor<Block>> for TestSync {
 	fn set_sync_fork_request(&self, _peers: Vec<PeerId>, _hash: Hash, _number: NumberFor<Block>) {}
 }
 
+#[derive(Debug)]
+pub(crate) struct TestNotificationService {
+	sender: TracingUnboundedSender<Event>,
+	rx: TracingUnboundedReceiver<NotificationEvent>,
+}
+
+#[async_trait::async_trait]
+impl NotificationService for TestNotificationService {
+	/// Instruct `Notifications` to open a new substream for `peer`.
+	async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> {
+		unimplemented!();
+	}
+
+	/// Instruct `Notifications` to close substream for `peer`.
+	async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> {
+		unimplemented!();
+	}
+
+	/// Send synchronous `notification` to `peer`.
+	fn send_sync_notification(&self, peer: &PeerId, notification: Vec<u8>) {
+		let _ = self.sender.unbounded_send(Event::WriteNotification(*peer, notification));
+	}
+
+	/// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure.
+	async fn send_async_notification(
+		&self,
+		_peer: &PeerId,
+		_notification: Vec<u8>,
+	) -> Result<(), sc_network::error::Error> {
+		unimplemented!();
+	}
+
+	/// Set handshake for the notification protocol replacing the old handshake.
+	async fn set_handshake(&mut self, _handshake: Vec<u8>) -> Result<(), ()> {
+		unimplemented!();
+	}
+
+	fn try_set_handshake(&mut self, _handshake: Vec<u8>) -> Result<(), ()> {
+		unimplemented!();
+	}
+
+	/// Get next event from the `Notifications` event stream.
+	async fn next_event(&mut self) -> Option<NotificationEvent> {
+		self.rx.next().await
+	}
+
+	fn clone(&mut self) -> Result<Box<dyn NotificationService>, ()> {
+		unimplemented!();
+	}
+
+	fn protocol(&self) -> &ProtocolName {
+		unimplemented!();
+	}
+
+	fn message_sink(&self, _peer: &PeerId) -> Option<Box<dyn MessageSink>> {
+		unimplemented!();
+	}
+}
+
 pub(crate) struct Tester {
 	pub(crate) net_handle: super::NetworkBridge<Block, TestNetwork, TestSync>,
 	gossip_validator: Arc<GossipValidator<Block>>,
 	pub(crate) events: TracingUnboundedReceiver<Event>,
+	pub(crate) notification_tx: TracingUnboundedSender<NotificationEvent>,
 }
 
 impl Tester {
@@ -279,6 +346,9 @@ fn voter_set_state() -> SharedVoterSetState<Block> {
 // needs to run in a tokio runtime.
 pub(crate) fn make_test_network() -> (impl Future<Output = Tester>, TestNetwork) {
 	let (tx, rx) = tracing_unbounded("test", 100_000);
+	let (notification_tx, notification_rx) = tracing_unbounded("test-notification", 100_000);
+
+	let notification_service = TestNotificationService { rx: notification_rx, sender: tx.clone() };
 	let net = TestNetwork { sender: tx };
 	let sync = TestSync {};
 
@@ -293,14 +363,22 @@ pub(crate) fn make_test_network() -> (impl Future<Output = Tester>, TestNetwork)
 		}
 	}
 
-	let bridge =
-		super::NetworkBridge::new(net.clone(), sync, config(), voter_set_state(), None, None);
+	let bridge = super::NetworkBridge::new(
+		net.clone(),
+		sync,
+		Box::new(notification_service),
+		config(),
+		voter_set_state(),
+		None,
+		None,
+	);
 
 	(
 		futures::future::ready(Tester {
 			gossip_validator: bridge.validator.clone(),
 			net_handle: bridge,
 			events: rx,
+			notification_tx,
 		}),
 		net,
 	)
@@ -385,63 +463,62 @@ fn good_commit_leads_to_relay() {
 			let commit_to_send = encoded_commit.clone();
 			let network_bridge = tester.net_handle.clone();
 
-			// asking for global communication will cause the test network
-			// to send us an event asking us for a stream. use it to
-			// send a message.
+			// `NetworkBridge` will be operational as soon as it's created and it's
+			// waiting for events from the network. Send it events that inform that
+			// a notification stream was opened and that a notification was received.
+			//
+			// Since each protocol has its own notification stream, events need not be filtered.
 			let sender_id = id;
-			let send_message = tester.filter_network_events(move |event| match event {
-				Event::EventStream(sender) => {
-					// Add the sending peer and send the commit
-					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
-						remote: sender_id,
-						protocol: grandpa_protocol_name::NAME.into(),
+
+			let send_message = async move {
+				let _ = tester.notification_tx.unbounded_send(
+					NotificationEvent::NotificationStreamOpened {
+						peer: sender_id,
+						direction: Direction::Inbound,
 						negotiated_fallback: None,
-						role: ObservedRole::Full,
-						received_handshake: vec![],
-					});
+						handshake: Roles::FULL.encode(),
+					},
+				);
+				let _ = tester.notification_tx.unbounded_send(
+					NotificationEvent::NotificationReceived {
+						peer: sender_id,
+						notification: commit_to_send.clone(),
+					},
+				);
 
-					let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived {
-						remote: sender_id,
-						messages: vec![(
-							grandpa_protocol_name::NAME.into(),
-							commit_to_send.clone().into(),
-						)],
-					});
-
-					// Add a random peer which will be the recipient of this message
-					let receiver_id = PeerId::random();
-					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
-						remote: receiver_id,
-						protocol: grandpa_protocol_name::NAME.into(),
+				// Add a random peer which will be the recipient of this message
+				let receiver_id = PeerId::random();
+				let _ = tester.notification_tx.unbounded_send(
+					NotificationEvent::NotificationStreamOpened {
+						peer: receiver_id,
+						direction: Direction::Inbound,
 						negotiated_fallback: None,
-						role: ObservedRole::Full,
-						received_handshake: vec![],
+						handshake: Roles::FULL.encode(),
+					},
+				);
+
+				// Announce its local set being on the current set id through a neighbor
+				// packet, otherwise it won't be eligible to receive the commit
+				let _ = {
+					let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket {
+						round: Round(round),
+						set_id: SetId(set_id),
+						commit_finalized_height: 1,
 					});
 
-					// Announce its local set has being on the current set id through a neighbor
-					// packet, otherwise it won't be eligible to receive the commit
-					let _ = {
-						let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket {
-							round: Round(round),
-							set_id: SetId(set_id),
-							commit_finalized_height: 1,
-						});
+					let msg = gossip::GossipMessage::<Block>::Neighbor(update);
 
-						let msg = gossip::GossipMessage::<Block>::Neighbor(update);
+					let _ = tester.notification_tx.unbounded_send(
+						NotificationEvent::NotificationReceived {
+							peer: receiver_id,
+							notification: msg.encode(),
+						},
+					);
+				};
 
-						sender.unbounded_send(NetworkEvent::NotificationsReceived {
-							remote: receiver_id,
-							messages: vec![(
-								grandpa_protocol_name::NAME.into(),
-								msg.encode().into(),
-							)],
-						})
-					};
-
-					true
-				},
-				_ => false,
-			});
+				tester
+			}
+			.boxed();
 
 			// when the commit comes in, we'll tell the callback it was good.
 			let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() {
@@ -537,31 +614,32 @@ fn bad_commit_leads_to_report() {
 			let commit_to_send = encoded_commit.clone();
 			let network_bridge = tester.net_handle.clone();
 
-			// asking for global communication will cause the test network
-			// to send us an event asking us for a stream. use it to
-			// send a message.
+			// `NetworkBridge` will be operational as soon as it's created and it's
+			// waiting for events from the network. Send it events that inform that
+			// a notification stream was opened and that a notification was received.
+			//
+			// Since each protocol has its own notification stream, events need not be filtered.
 			let sender_id = id;
-			let send_message = tester.filter_network_events(move |event| match event {
-				Event::EventStream(sender) => {
-					let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened {
-						remote: sender_id,
-						protocol: grandpa_protocol_name::NAME.into(),
-						negotiated_fallback: None,
-						role: ObservedRole::Full,
-						received_handshake: vec![],
-					});
-					let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived {
-						remote: sender_id,
-						messages: vec![(
-							grandpa_protocol_name::NAME.into(),
-							commit_to_send.clone().into(),
-						)],
-					});
 
-					true
-				},
-				_ => false,
-			});
+			let send_message = async move {
+				let _ = tester.notification_tx.unbounded_send(
+					NotificationEvent::NotificationStreamOpened {
+						peer: sender_id,
+						direction: Direction::Inbound,
+						negotiated_fallback: None,
+						handshake: Roles::FULL.encode(),
+					},
+				);
+				let _ = tester.notification_tx.unbounded_send(
+					NotificationEvent::NotificationReceived {
+						peer: sender_id,
+						notification: commit_to_send.clone(),
+					},
+				);
+
+				tester
+			}
+			.boxed();
 
 			// when the commit comes in, we'll tell the callback it was bad.
 			let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() {

substrate/client/consensus/grandpa/src/lib.rs

@@ -67,7 +67,7 @@ use sc_client_api::{
 	BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, StorageProvider,
 };
 use sc_consensus::BlockImport;
-use sc_network::types::ProtocolName;
+use sc_network::{types::ProtocolName, NotificationService};
 use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO};
 use sc_transaction_pool_api::OffchainTransactionPoolFactory;
 use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
@@ -687,6 +687,8 @@ pub struct GrandpaParams<Block: BlockT, C, N, S, SC, VR> {
 	pub network: N,
 	/// Event stream for syncing-related events.
 	pub sync: S,
+	/// Handle for interacting with `Notifications`.
+	pub notification_service: Box<dyn NotificationService>,
 	/// A voting rule used to potentially restrict target votes.
 	pub voting_rule: VR,
 	/// The prometheus metrics registry.
@@ -707,21 +709,21 @@ pub struct GrandpaParams<Block: BlockT, C, N, S, SC, VR> {
 /// For standard protocol name see [`crate::protocol_standard_name`].
 pub fn grandpa_peers_set_config(
 	protocol_name: ProtocolName,
-) -> sc_network::config::NonDefaultSetConfig {
+) -> (sc_network::config::NonDefaultSetConfig, Box<dyn NotificationService>) {
 	use communication::grandpa_protocol_name;
-	sc_network::config::NonDefaultSetConfig {
-		notifications_protocol: protocol_name,
-		fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(),
+	sc_network::config::NonDefaultSetConfig::new(
+		protocol_name,
+		grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(),
 		// Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot.
-		max_notification_size: 1024 * 1024,
-		handshake: None,
-		set_config: sc_network::config::SetConfig {
+		1024 * 1024,
+		None,
+		sc_network::config::SetConfig {
 			in_peers: 0,
 			out_peers: 0,
 			reserved_nodes: Vec::new(),
 			non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny,
 		},
-	}
+	)
 }
 
 /// Run a GRANDPA voter as a task. Provide configuration and a link to a
@@ -744,6 +746,7 @@ where
 		link,
 		network,
 		sync,
+		notification_service,
 		voting_rule,
 		prometheus_registry,
 		shared_voter_state,
@@ -770,6 +773,7 @@ where
 	let network = NetworkBridge::new(
 		network,
 		sync,
+		notification_service,
 		config.clone(),
 		persistent_data.set_state.clone(),
 		prometheus_registry.as_ref(),

substrate/client/consensus/grandpa/src/observer.rs

@@ -28,6 +28,7 @@ use futures::prelude::*;
 use log::{debug, info, warn};
 
 use sc_client_api::backend::Backend;
+use sc_network::NotificationService;
 use sc_telemetry::TelemetryHandle;
 use sc_utils::mpsc::TracingUnboundedReceiver;
 use sp_blockchain::HeaderMetadata;
@@ -168,6 +169,7 @@ pub fn run_grandpa_observer<BE, Block: BlockT, Client, N, S, SC>(
 	link: LinkHalf<Block, Client, SC>,
 	network: N,
 	sync: S,
+	notification_service: Box<dyn NotificationService>,
 ) -> sp_blockchain::Result<impl Future<Output = ()> + Send>
 where
 	BE: Backend<Block> + Unpin + 'static,
@@ -189,6 +191,7 @@ where
 	let network = NetworkBridge::new(
 		network,
 		sync,
+		notification_service,
 		config.clone(),
 		persistent_data.set_state.clone(),
 		None,
@@ -414,14 +417,14 @@ mod tests {
 
 	use futures::executor;
 
-	/// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. Regression
-	/// test for bug introduced in d4fbb897c and fixed in b7af8b339.
+	/// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`.
+	/// Regression test for bug introduced in d4fbb897c and fixed in b7af8b339.
 	///
-	/// When polled, `NetworkBridge` forwards reputation change requests from the `GossipValidator`
-	/// to the underlying `dyn Network`. This test triggers a reputation change by calling
-	/// `GossipValidator::validate` with an invalid gossip message. After polling the `ObserverWork`
-	/// which should poll the `NetworkBridge`, the reputation change should be forwarded to the test
-	/// network.
+	/// When polled, `NetworkBridge` forwards reputation change requests from the
+	/// `GossipValidator` to the underlying `dyn Network`. This test triggers a reputation change
+	/// by calling `GossipValidator::validate` with an invalid gossip message. After polling the
+	/// `ObserverWork` which should poll the `NetworkBridge`, the reputation change should be
+	/// forwarded to the test network.
 	#[test]
 	fn observer_work_polls_underlying_network_bridge() {
 		// Create a test network.
@@ -463,12 +466,6 @@ mod tests {
 			// validator to the test network.
 			assert!(observer.now_or_never().is_none());
 
-			// Ignore initial event stream request by gossip engine.
-			match tester.events.next().now_or_never() {
-				Some(Some(Event::EventStream(_))) => {},
-				_ => panic!("expected event stream request"),
-			};
-
 			assert_matches!(tester.events.next().now_or_never(), Some(Some(Event::Report(_, _))));
 		});
 	}

substrate/client/consensus/grandpa/src/tests.rs

@@ -317,6 +317,9 @@ fn initialize_grandpa(
 			(net.peers[peer_id].network_service().clone(), link)
 		};
 		let sync = net.peers[peer_id].sync_service().clone();
+		let notification_service = net.peers[peer_id]
+			.take_notification_service(&grandpa_protocol_name::NAME.into())
+			.unwrap();
 
 		let grandpa_params = GrandpaParams {
 			config: Config {
@@ -332,6 +335,7 @@ fn initialize_grandpa(
 			link,
 			network: net_service,
 			sync,
+			notification_service,
 			voting_rule: (),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -472,6 +476,9 @@ async fn finalize_3_voters_1_full_observer() {
 		let net_service = net.peers[peer_id].network_service().clone();
 		let sync = net.peers[peer_id].sync_service().clone();
 		let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed");
+		let notification_service = net.peers[peer_id]
+			.take_notification_service(&grandpa_protocol_name::NAME.into())
+			.unwrap();
 
 		let grandpa_params = GrandpaParams {
 			config: Config {
@@ -487,6 +494,7 @@ async fn finalize_3_voters_1_full_observer() {
 			link,
 			network: net_service,
 			sync,
+			notification_service,
 			voting_rule: (),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -557,14 +565,17 @@ async fn transition_3_voters_twice_1_full_observer() {
 	for (peer_id, local_key) in all_peers.clone().into_iter().enumerate() {
 		let keystore = create_keystore(local_key);
 
-		let (net_service, link, sync) = {
-			let net = net.lock();
+		let (net_service, link, sync, notification_service) = {
+			let mut net = net.lock();
 			let link =
 				net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed");
 			(
 				net.peers[peer_id].network_service().clone(),
 				link,
 				net.peers[peer_id].sync_service().clone(),
+				net.peers[peer_id]
+					.take_notification_service(&grandpa_protocol_name::NAME.into())
+					.unwrap(),
 			)
 		};
 
@@ -582,6 +593,7 @@ async fn transition_3_voters_twice_1_full_observer() {
 			link,
 			network: net_service,
 			sync,
+			notification_service,
 			voting_rule: (),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -1025,6 +1037,9 @@ async fn voter_persists_its_votes() {
 		communication::NetworkBridge::new(
 			net.peers[1].network_service().clone(),
 			net.peers[1].sync_service().clone(),
+			net.peers[1]
+				.take_notification_service(&grandpa_protocol_name::NAME.into())
+				.unwrap(),
 			config.clone(),
 			set_state,
 			None,
@@ -1043,6 +1058,9 @@ async fn voter_persists_its_votes() {
 			(net.peers[0].network_service().clone(), link)
 		};
 		let sync = net.peers[0].sync_service().clone();
+		let notification_service = net.peers[0]
+			.take_notification_service(&grandpa_protocol_name::NAME.into())
+			.unwrap();
 
 		let grandpa_params = GrandpaParams {
 			config: Config {
@@ -1058,6 +1076,7 @@ async fn voter_persists_its_votes() {
 			link,
 			network: net_service,
 			sync,
+			notification_service,
 			voting_rule: VotingRulesBuilder::default().build(),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -1082,6 +1101,9 @@ async fn voter_persists_its_votes() {
 		net.add_authority_peer();
 		let net_service = net.peers[2].network_service().clone();
 		let sync = net.peers[2].sync_service().clone();
+		let notification_service = net.peers[2]
+			.take_notification_service(&grandpa_protocol_name::NAME.into())
+			.unwrap();
 		// but we'll reuse the client from the first peer (alice_voter1)
 		// since we want to share the same database, so that we can
 		// read the persisted state after aborting alice_voter1.
@@ -1104,6 +1126,7 @@ async fn voter_persists_its_votes() {
 			link,
 			network: net_service,
 			sync,
+			notification_service,
 			voting_rule: VotingRulesBuilder::default().build(),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -1255,6 +1278,9 @@ async fn finalize_3_voters_1_light_observer() {
 
 	let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 1);
 	let voters = initialize_grandpa(&mut net, authorities);
+	let notification_service = net.peers[3]
+		.take_notification_service(&grandpa_protocol_name::NAME.into())
+		.unwrap();
 	let observer = observer::run_grandpa_observer(
 		Config {
 			gossip_duration: TEST_GOSSIP_DURATION,
@@ -1269,6 +1295,7 @@ async fn finalize_3_voters_1_light_observer() {
 		net.peers[3].data.lock().take().expect("link initialized at startup; qed"),
 		net.peers[3].network_service().clone(),
 		net.peers[3].sync_service().clone(),
+		notification_service,
 	)
 	.unwrap();
 	net.peer(0).push_blocks(20, false);
@@ -1317,6 +1344,10 @@ async fn voter_catches_up_to_latest_round_when_behind() {
 			link,
 			network: net.peer(peer_id).network_service().clone(),
 			sync: net.peer(peer_id).sync_service().clone(),
+			notification_service: net
+				.peer(peer_id)
+				.take_notification_service(&grandpa_protocol_name::NAME.into())
+				.unwrap(),
 			voting_rule: (),
 			prometheus_registry: None,
 			shared_voter_state: SharedVoterState::empty(),
@@ -1409,6 +1440,7 @@ fn test_environment_with_select_chain<N, S, VR, SC>(
 	keystore: Option<KeystorePtr>,
 	network_service: N,
 	sync_service: S,
+	notification_service: Box<dyn NotificationService>,
 	select_chain: SC,
 	voting_rule: VR,
 ) -> TestEnvironment<N, S, SC, VR>
@@ -1433,6 +1465,7 @@ where
 	let network = NetworkBridge::new(
 		network_service.clone(),
 		sync_service,
+		notification_service,
 		config.clone(),
 		set_state.clone(),
 		None,
@@ -1462,6 +1495,7 @@ fn test_environment<N, S, VR>(
 	keystore: Option<KeystorePtr>,
 	network_service: N,
 	sync_service: S,
+	notification_service: Box<dyn NotificationService>,
 	voting_rule: VR,
 ) -> TestEnvironment<N, S, LongestChain<substrate_test_runtime_client::Backend, Block>, VR>
 where
@@ -1474,6 +1508,7 @@ where
 		keystore,
 		network_service,
 		sync_service,
+		notification_service,
 		link.select_chain.clone(),
 		voting_rule,
 	)
@@ -1490,14 +1525,22 @@ async fn grandpa_environment_respects_voting_rules() {
 	let peer = net.peer(0);
 	let network_service = peer.network_service().clone();
 	let sync_service = peer.sync_service().clone();
+	let mut notification_service =
+		peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap();
 	let link = peer.data.lock().take().unwrap();
 
 	// add 21 blocks
 	let hashes = peer.push_blocks(21, false);
 
 	// create an environment with no voting rule restrictions
-	let unrestricted_env =
-		test_environment(&link, None, network_service.clone(), sync_service.clone(), ());
+	let unrestricted_env = test_environment(
+		&link,
+		None,
+		network_service.clone(),
+		sync_service.clone(),
+		notification_service.clone().unwrap(),
+		(),
+	);
 
 	// another with 3/4 unfinalized chain voting rule restriction
 	let three_quarters_env = test_environment(
@@ -1505,6 +1548,7 @@ async fn grandpa_environment_respects_voting_rules() {
 		None,
 		network_service.clone(),
 		sync_service.clone(),
+		notification_service.clone().unwrap(),
 		voting_rule::ThreeQuartersOfTheUnfinalizedChain,
 	);
 
@@ -1515,6 +1559,7 @@ async fn grandpa_environment_respects_voting_rules() {
 		None,
 		network_service.clone(),
 		sync_service,
+		notification_service,
 		VotingRulesBuilder::default().build(),
 	);
 
@@ -1608,6 +1653,8 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() {
 	let peer = net.peer(0);
 	let network_service = peer.network_service().clone();
 	let sync_service = peer.sync_service().clone();
+	let notification_service =
+		peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap();
 	let link = peer.data.lock().take().unwrap();
 	let client = peer.client().as_client().clone();
 	let select_chain = MockSelectChain::default();
@@ -1622,6 +1669,7 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() {
 		None,
 		network_service.clone(),
 		sync_service,
+		notification_service,
 		select_chain.clone(),
 		voting_rule::BeforeBestBlockBy(5),
 	);
@@ -1669,6 +1717,8 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ
 	let peer = net.peer(0);
 	let network_service = peer.network_service().clone();
 	let sync_service = peer.sync_service().clone();
+	let notification_service =
+		peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap();
 	let link = peer.data.lock().take().unwrap();
 	let client = peer.client().as_client().clone();
 	let select_chain = MockSelectChain::default();
@@ -1678,6 +1728,7 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ
 		None,
 		network_service.clone(),
 		sync_service.clone(),
+		notification_service,
 		select_chain.clone(),
 		voting_rule.clone(),
 	);
@@ -1780,11 +1831,19 @@ async fn grandpa_environment_never_overwrites_round_voter_state() {
 	let peer = net.peer(0);
 	let network_service = peer.network_service().clone();
 	let sync_service = peer.sync_service().clone();
+	let notification_service =
+		peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap();
 	let link = peer.data.lock().take().unwrap();
 
 	let keystore = create_keystore(peers[0]);
-	let environment =
-		test_environment(&link, Some(keystore), network_service.clone(), sync_service, ());
+	let environment = test_environment(
+		&link,
+		Some(keystore),
+		network_service.clone(),
+		sync_service,
+		notification_service,
+		(),
+	);
 
 	let round_state = || finality_grandpa::round::State::genesis(Default::default());
 	let base = || Default::default();
@@ -2012,9 +2071,18 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() {
 		let peer = net.peer(0);
 		let network_service = peer.network_service().clone();
 		let sync_service = peer.sync_service().clone();
+		let notification_service =
+			peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap();
 		let link = peer.data.lock().take().unwrap();
 		let keystore = create_keystore(alice);
-		test_environment(&link, Some(keystore), network_service.clone(), sync_service, ())
+		test_environment(
+			&link,
+			Some(keystore),
+			network_service.clone(),
+			sync_service,
+			notification_service,
+			(),
+		)
 	};
 
 	let signed_prevote = {