Rework the event system of sc-network (#1370)

This commit introduces a new concept called `NotificationService` which allows Polkadot protocols to communicate with the underlying notification protocol implementation directly, without routing events through `NetworkWorker`. This implies that each protocol has its own service which it uses to communicate with remote peers and that each `NotificationService` is unique with respect to the underlying notification protocol, meaning `NotificationService` for the transaction protocol can only be used to send and receive transaction-related notifications. The `NotificationService` concept introduces two additional benefits: * allow protocols to start using custom handshakes * allow protocols to accept/reject inbound peers Previously the validation of inbound connections was solely the responsibility of `ProtocolController`. This caused issues with light peers and `SyncingEngine` as `ProtocolController` would accept more peers than `SyncingEngine` could accept which caused peers to have differing views of their own states. `SyncingEngine` would reject excess peers but these rejections were not properly communicated to those peers causing them to assume that they were accepted. With `NotificationService`, the local handshake is not sent to remote peer if peer is rejected which allows it to detect that it was rejected. This commit also deprecates the use of `NetworkEventStream` for all notification-related events and going forward only DHT events are provided through `NetworkEventStream`. If protocols wish to follow each other's events, they must introduce additional abtractions, as is done for GRANDPA and transactions protocols by following the syncing protocol through `SyncEventStream`. Fixes https://github.com/paritytech/polkadot-sdk/issues/512 Fixes https://github.com/paritytech/polkadot-sdk/issues/514 Fixes https://github.com/paritytech/polkadot-sdk/issues/515 Fixes https://github.com/paritytech/polkadot-sdk/issues/554 Fixes https://github.com/paritytech/polkadot-sdk/issues/556 --- These changes are transferred from https://github.com/paritytech/substrate/pull/14197 but there are no functional changes compared to that PR --------- Co-authored-by: Dmitry Markin <dmitry@markin.tech> Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
2026-06-23 14:21:11 +00:00 · 2023-11-28 20:18:52 +02:00
parent ec3a61ed86
commit e71c484d5b
102 changed files with 5694 additions and 2603 deletions
@@ -23,7 +23,7 @@ use libp2p::PeerId;
 use schnellru::{ByLength, LruMap};

 use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64};
-use sc_network::types::ProtocolName;
+use sc_network::{types::ProtocolName, NotificationService};
 use sc_network_common::role::ObservedRole;
 use sp_runtime::traits::{Block as BlockT, Hash, HashingFor};
 use std::{collections::HashMap, iter, sync::Arc, time, time::Instant};
@@ -74,33 +74,33 @@ struct MessageEntry<B: BlockT> {
 /// Local implementation of `ValidatorContext`.
 struct NetworkContext<'g, 'p, B: BlockT> {
 	gossip: &'g mut ConsensusGossip<B>,
-	network: &'p mut dyn Network<B>,
+	notification_service: &'p mut Box<dyn NotificationService>,
 }

 impl<'g, 'p, B: BlockT> ValidatorContext<B> for NetworkContext<'g, 'p, B> {
 	/// Broadcast all messages with given topic to peers that do not have it yet.
 	fn broadcast_topic(&mut self, topic: B::Hash, force: bool) {
-		self.gossip.broadcast_topic(self.network, topic, force);
+		self.gossip.broadcast_topic(self.notification_service, topic, force);
 	}

 	/// Broadcast a message to all peers that have not received it previously.
 	fn broadcast_message(&mut self, topic: B::Hash, message: Vec<u8>, force: bool) {
-		self.gossip.multicast(self.network, topic, message, force);
+		self.gossip.multicast(self.notification_service, topic, message, force);
 	}

 	/// Send addressed message to a peer.
 	fn send_message(&mut self, who: &PeerId, message: Vec<u8>) {
-		self.network.write_notification(*who, self.gossip.protocol.clone(), message);
+		self.notification_service.send_sync_notification(who, message);
 	}

 	/// Send all messages with given topic to a peer.
 	fn send_topic(&mut self, who: &PeerId, topic: B::Hash, force: bool) {
-		self.gossip.send_topic(self.network, who, topic, force);
+		self.gossip.send_topic(self.notification_service, who, topic, force);
 	}
 }

 fn propagate<'a, B: BlockT, I>(
-	network: &mut dyn Network<B>,
+	notification_service: &mut Box<dyn NotificationService>,
 	protocol: ProtocolName,
 	messages: I,
 	intent: MessageIntent,
@@ -147,7 +147,7 @@ where
 				?message,
 				"Propagating message",
 			);
-			network.write_notification(*id, protocol.clone(), message.clone());
+			notification_service.send_sync_notification(id, message.clone());
 		}
 	}
 }
@@ -191,7 +191,12 @@ impl<B: BlockT> ConsensusGossip<B> {
 	}

 	/// Handle new connected peer.
-	pub fn new_peer(&mut self, network: &mut dyn Network<B>, who: PeerId, role: ObservedRole) {
+	pub fn new_peer(
+		&mut self,
+		notification_service: &mut Box<dyn NotificationService>,
+		who: PeerId,
+		role: ObservedRole,
+	) {
 		tracing::trace!(
 			target:"gossip",
 			%who,
@@ -202,7 +207,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 		self.peers.insert(who, PeerConsensus { known_messages: Default::default() });

 		let validator = self.validator.clone();
-		let mut context = NetworkContext { gossip: self, network };
+		let mut context = NetworkContext { gossip: self, notification_service };
 		validator.new_peer(&mut context, &who, role);
 	}

@@ -233,30 +238,35 @@ impl<B: BlockT> ConsensusGossip<B> {
 	}

 	/// Call when a peer has been disconnected to stop tracking gossip status.
-	pub fn peer_disconnected(&mut self, network: &mut dyn Network<B>, who: PeerId) {
+	pub fn peer_disconnected(
+		&mut self,
+		notification_service: &mut Box<dyn NotificationService>,
+		who: PeerId,
+	) {
 		let validator = self.validator.clone();
-		let mut context = NetworkContext { gossip: self, network };
+		let mut context = NetworkContext { gossip: self, notification_service };
 		validator.peer_disconnected(&mut context, &who);
 		self.peers.remove(&who);
 	}

 	/// Perform periodic maintenance
-	pub fn tick(&mut self, network: &mut dyn Network<B>) {
+	pub fn tick(&mut self, notification_service: &mut Box<dyn NotificationService>) {
 		self.collect_garbage();
 		if Instant::now() >= self.next_broadcast {
-			self.rebroadcast(network);
+			self.rebroadcast(notification_service);
 			self.next_broadcast = Instant::now() + REBROADCAST_INTERVAL;
 		}
 	}

 	/// Rebroadcast all messages to all peers.
-	fn rebroadcast(&mut self, network: &mut dyn Network<B>) {
+	fn rebroadcast(&mut self, notification_service: &mut Box<dyn NotificationService>) {
 		let messages = self
 			.messages
 			.iter()
 			.map(|entry| (&entry.message_hash, &entry.topic, &entry.message));
+
 		propagate(
-			network,
+			notification_service,
 			self.protocol.clone(),
 			messages,
 			MessageIntent::PeriodicRebroadcast,
@@ -266,7 +276,12 @@ impl<B: BlockT> ConsensusGossip<B> {
 	}

 	/// Broadcast all messages with given topic.
-	pub fn broadcast_topic(&mut self, network: &mut dyn Network<B>, topic: B::Hash, force: bool) {
+	pub fn broadcast_topic(
+		&mut self,
+		notification_service: &mut Box<dyn NotificationService>,
+		topic: B::Hash,
+		force: bool,
+	) {
 		let messages = self.messages.iter().filter_map(|entry| {
 			if entry.topic == topic {
 				Some((&entry.message_hash, &entry.topic, &entry.message))
@@ -276,7 +291,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 		});
 		let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast };
 		propagate(
-			network,
+			notification_service,
 			self.protocol.clone(),
 			messages,
 			intent,
@@ -327,6 +342,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 	pub fn on_incoming(
 		&mut self,
 		network: &mut dyn Network<B>,
+		notification_service: &mut Box<dyn NotificationService>,
 		who: PeerId,
 		messages: Vec<Vec<u8>>,
 	) -> Vec<(B::Hash, TopicNotification)> {
@@ -367,7 +383,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 			// validate the message
 			let validation = {
 				let validator = self.validator.clone();
-				let mut context = NetworkContext { gossip: self, network };
+				let mut context = NetworkContext { gossip: self, notification_service };
 				validator.validate(&mut context, &who, &message)
 			};

@@ -414,7 +430,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 	/// Send all messages with given topic to a peer.
 	pub fn send_topic(
 		&mut self,
-		network: &mut dyn Network<B>,
+		notification_service: &mut Box<dyn NotificationService>,
 		who: &PeerId,
 		topic: B::Hash,
 		force: bool,
@@ -443,7 +459,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 					?entry.message,
 					"Sending topic message",
 				);
-				network.write_notification(*who, self.protocol.clone(), entry.message.clone());
+				notification_service.send_sync_notification(who, entry.message.clone());
 			}
 		}
 	}
@@ -451,7 +467,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 	/// Multicast a message to all peers.
 	pub fn multicast(
 		&mut self,
-		network: &mut dyn Network<B>,
+		notification_service: &mut Box<dyn NotificationService>,
 		topic: B::Hash,
 		message: Vec<u8>,
 		force: bool,
@@ -460,7 +476,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 		self.register_message_hashed(message_hash, topic, message.clone(), None);
 		let intent = if force { MessageIntent::ForcedBroadcast } else { MessageIntent::Broadcast };
 		propagate(
-			network,
+			notification_service,
 			self.protocol.clone(),
 			iter::once((&message_hash, &topic, &message)),
 			intent,
@@ -471,7 +487,12 @@ impl<B: BlockT> ConsensusGossip<B> {

 	/// Send addressed message to a peer. The message is not kept or multicast
 	/// later on.
-	pub fn send_message(&mut self, network: &mut dyn Network<B>, who: &PeerId, message: Vec<u8>) {
+	pub fn send_message(
+		&mut self,
+		notification_service: &mut Box<dyn NotificationService>,
+		who: &PeerId,
+		message: Vec<u8>,
+	) {
 		let peer = match self.peers.get_mut(who) {
 			None => return,
 			Some(peer) => peer,
@@ -488,7 +509,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 		);

 		peer.known_messages.insert(message_hash);
-		network.write_notification(*who, self.protocol.clone(), message);
+		notification_service.send_sync_notification(who, message)
 	}
 }

@@ -524,9 +545,9 @@ mod tests {
 	use crate::multiaddr::Multiaddr;
 	use futures::prelude::*;
 	use sc_network::{
-		config::MultiaddrWithPeerId, event::Event, NetworkBlock, NetworkEventStream,
-		NetworkNotification, NetworkPeers, NotificationSenderError,
-		NotificationSenderT as NotificationSender, ReputationChange,
+		config::MultiaddrWithPeerId, event::Event, service::traits::NotificationEvent, MessageSink,
+		NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers,
+		NotificationSenderError, NotificationSenderT as NotificationSender, ReputationChange,
 	};
 	use sp_runtime::{
 		testing::{Block as RawBlock, ExtrinsicWrapper, H256},
@@ -651,6 +672,10 @@ mod tests {
 		fn sync_num_connected(&self) -> usize {
 			unimplemented!();
 		}
+
+		fn peer_role(&self, _peer_id: PeerId, _handshake: Vec<u8>) -> Option<ObservedRole> {
+			None
+		}
 	}

 	impl NetworkEventStream for NoOpNetwork {
@@ -691,6 +716,62 @@ mod tests {
 		}
 	}

+	#[derive(Debug, Default)]
+	struct NoOpNotificationService {}
+
+	#[async_trait::async_trait]
+	impl NotificationService for NoOpNotificationService {
+		/// Instruct `Notifications` to open a new substream for `peer`.
+		async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> {
+			unimplemented!();
+		}
+
+		/// Instruct `Notifications` to close substream for `peer`.
+		async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> {
+			unimplemented!();
+		}
+
+		/// Send synchronous `notification` to `peer`.
+		fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec<u8>) {
+			unimplemented!();
+		}
+
+		/// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure.
+		async fn send_async_notification(
+			&self,
+			_peer: &PeerId,
+			_notification: Vec<u8>,
+		) -> Result<(), sc_network::error::Error> {
+			unimplemented!();
+		}
+
+		/// Set handshake for the notification protocol replacing the old handshake.
+		async fn set_handshake(&mut self, _handshake: Vec<u8>) -> Result<(), ()> {
+			unimplemented!();
+		}
+
+		fn try_set_handshake(&mut self, _handshake: Vec<u8>) -> Result<(), ()> {
+			unimplemented!();
+		}
+
+		/// Get next event from the `Notifications` event stream.
+		async fn next_event(&mut self) -> Option<NotificationEvent> {
+			None
+		}
+
+		fn clone(&mut self) -> Result<Box<dyn NotificationService>, ()> {
+			unimplemented!();
+		}
+
+		fn protocol(&self) -> &ProtocolName {
+			unimplemented!();
+		}
+
+		fn message_sink(&self, _peer: &PeerId) -> Option<Box<dyn MessageSink>> {
+			unimplemented!();
+		}
+	}
+
 	#[test]
 	fn collects_garbage() {
 		struct AllowOne;
@@ -773,20 +854,28 @@ mod tests {
 	fn peer_is_removed_on_disconnect() {
 		let mut consensus = ConsensusGossip::<Block>::new(Arc::new(AllowAll), "/foo".into(), None);

-		let mut network = NoOpNetwork::default();
+		let mut notification_service: Box<dyn NotificationService> =
+			Box::new(NoOpNotificationService::default());

 		let peer_id = PeerId::random();
-		consensus.new_peer(&mut network, peer_id, ObservedRole::Full);
+		consensus.new_peer(&mut notification_service, peer_id, ObservedRole::Full);
 		assert!(consensus.peers.contains_key(&peer_id));

-		consensus.peer_disconnected(&mut network, peer_id);
+		consensus.peer_disconnected(&mut notification_service, peer_id);
 		assert!(!consensus.peers.contains_key(&peer_id));
 	}

 	#[test]
 	fn on_incoming_ignores_discarded_messages() {
+		let mut notification_service: Box<dyn NotificationService> =
+			Box::new(NoOpNotificationService::default());
 		let to_forward = ConsensusGossip::<Block>::new(Arc::new(DiscardAll), "/foo".into(), None)
-			.on_incoming(&mut NoOpNetwork::default(), PeerId::random(), vec![vec![1, 2, 3]]);
+			.on_incoming(
+				&mut NoOpNetwork::default(),
+				&mut notification_service,
+				PeerId::random(),
+				vec![vec![1, 2, 3]],
+			);

 		assert!(
 			to_forward.is_empty(),
@@ -798,11 +887,14 @@ mod tests {
 	#[test]
 	fn on_incoming_ignores_unregistered_peer() {
 		let mut network = NoOpNetwork::default();
+		let mut notification_service: Box<dyn NotificationService> =
+			Box::new(NoOpNotificationService::default());
 		let remote = PeerId::random();

 		let to_forward = ConsensusGossip::<Block>::new(Arc::new(AllowAll), "/foo".into(), None)
 			.on_incoming(
 				&mut network,
+				&mut notification_service,
 				// Unregistered peer.
 				remote,
 				vec![vec![1, 2, 3]],
@@ -822,18 +914,20 @@ mod tests {
 		let mut consensus = ConsensusGossip::<Block>::new(Arc::new(AllowAll), "/foo".into(), None);

 		let mut network = NoOpNetwork::default();
+		let mut notification_service: Box<dyn NotificationService> =
+			Box::new(NoOpNotificationService::default());

 		let peer_id = PeerId::random();
-		consensus.new_peer(&mut network, peer_id, ObservedRole::Full);
+		consensus.new_peer(&mut notification_service, peer_id, ObservedRole::Full);
 		assert!(consensus.peers.contains_key(&peer_id));

 		let peer_id2 = PeerId::random();
-		consensus.new_peer(&mut network, peer_id2, ObservedRole::Full);
+		consensus.new_peer(&mut notification_service, peer_id2, ObservedRole::Full);
 		assert!(consensus.peers.contains_key(&peer_id2));

 		let message = vec![vec![1, 2, 3]];
-		consensus.on_incoming(&mut network, peer_id, message.clone());
-		consensus.on_incoming(&mut network, peer_id2, message.clone());
+		consensus.on_incoming(&mut network, &mut notification_service, peer_id, message.clone());
+		consensus.on_incoming(&mut network, &mut notification_service, peer_id2, message.clone());

 		assert_eq!(
 			vec![(peer_id, rep::GOSSIP_SUCCESS)],