Rework the event system of sc-network (#1370)

This commit introduces a new concept called `NotificationService` which allows Polkadot protocols to communicate with the underlying notification protocol implementation directly, without routing events through `NetworkWorker`. This implies that each protocol has its own service which it uses to communicate with remote peers and that each `NotificationService` is unique with respect to the underlying notification protocol, meaning `NotificationService` for the transaction protocol can only be used to send and receive transaction-related notifications. The `NotificationService` concept introduces two additional benefits: * allow protocols to start using custom handshakes * allow protocols to accept/reject inbound peers Previously the validation of inbound connections was solely the responsibility of `ProtocolController`. This caused issues with light peers and `SyncingEngine` as `ProtocolController` would accept more peers than `SyncingEngine` could accept which caused peers to have differing views of their own states. `SyncingEngine` would reject excess peers but these rejections were not properly communicated to those peers causing them to assume that they were accepted. With `NotificationService`, the local handshake is not sent to remote peer if peer is rejected which allows it to detect that it was rejected. This commit also deprecates the use of `NetworkEventStream` for all notification-related events and going forward only DHT events are provided through `NetworkEventStream`. If protocols wish to follow each other's events, they must introduce additional abtractions, as is done for GRANDPA and transactions protocols by following the syncing protocol through `SyncEventStream`. Fixes https://github.com/paritytech/polkadot-sdk/issues/512 Fixes https://github.com/paritytech/polkadot-sdk/issues/514 Fixes https://github.com/paritytech/polkadot-sdk/issues/515 Fixes https://github.com/paritytech/polkadot-sdk/issues/554 Fixes https://github.com/paritytech/polkadot-sdk/issues/556 --- These changes are transferred from https://github.com/paritytech/substrate/pull/14197 but there are no functional changes compared to that PR --------- Co-authored-by: Dmitry Markin <dmitry@markin.tech> Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
2026-06-13 22:11:06 +00:00 · 2023-11-28 20:18:52 +02:00
parent ec3a61ed86
commit e71c484d5b
102 changed files with 5694 additions and 2603 deletions
@@ -24,7 +24,7 @@ use libp2p_identity::PeerId;
 use log::{debug, warn};
 use mixnet::core::{AddressedPacket, NetworkStatus, Packet, PeerId as CorePeerId};
 use parking_lot::Mutex;
-use sc_network::{NetworkNotification, ProtocolName};
+use sc_network::NotificationService;
 use std::{collections::HashMap, future::Future, sync::Arc};

 const LOG_TARGET: &str = "mixnet";
@@ -77,41 +77,37 @@ pub struct ReadyPeer {
 }

 impl ReadyPeer {
-	/// If a future is returned, and if that future returns `Some`, this function should be called
-	/// again to send the next packet queued for the peer; `self` is placed in the `Some` to make
-	/// this straightforward. Otherwise, we have either sent or dropped all packets queued for the
-	/// peer, and it can be forgotten about for the time being.
+	/// If a future is returned, and if that future returns `Some`, this function should be
+	/// called again to send the next packet queued for the peer; `self` is placed in the `Some`
+	/// to make this straightforward. Otherwise, we have either sent or dropped all packets
+	/// queued for the peer, and it can be forgotten about for the time being.
 	pub fn send_packet(
 		self,
-		network: &impl NetworkNotification,
-		protocol_name: ProtocolName,
+		notification_service: &Box<dyn NotificationService>,
 	) -> Option<impl Future<Output = Option<Self>>> {
-		match network.notification_sender(self.id, protocol_name) {
-			Err(err) => {
+		match notification_service.message_sink(&self.id) {
+			None => {
 				debug!(
 					target: LOG_TARGET,
-					"Failed to get notification sender for peer ID {}: {err}", self.id
+					"Failed to get message sink for peer ID {}", self.id,
 				);
 				self.queue.clear();
 				None
 			},
-			Ok(sender) => Some(async move {
-				match sender.ready().await.and_then(|mut ready| {
-					let (packet, more_packets) = self.queue.pop();
-					let packet =
-						packet.expect("Should only be called if there is a packet to send");
-					ready.send((packet as Box<[_]>).into())?;
-					Ok(more_packets)
-				}) {
+			Some(sink) => Some(async move {
+				let (packet, more_packets) = self.queue.pop();
+				let packet = packet.expect("Should only be called if there is a packet to send");
+
+				match sink.send_async_notification((packet as Box<[_]>).into()).await {
+					Ok(_) => more_packets.then_some(self),
 					Err(err) => {
 						debug!(
 							target: LOG_TARGET,
-							"Notification sender for peer ID {} failed: {err}", self.id
+							"Failed to send packet to peer ID {}: {err}", self.id,
 						);
 						self.queue.clear();
 						None
 					},
-					Ok(more_packets) => more_packets.then(|| self),
 				}
 			}),
 		}
@@ -18,7 +18,10 @@

 use super::config::Config;
 use mixnet::core::PACKET_SIZE;
-use sc_network::{config::NonDefaultSetConfig, ProtocolName};
+use sc_network::{
+	config::{NonDefaultSetConfig, NonReservedPeerMode, SetConfig},
+	NotificationService, ProtocolName,
+};

 /// Returns the protocol name to use for the mixnet controlled by the given chain.
 pub fn protocol_name(genesis_hash: &[u8], fork_id: Option<&str>) -> ProtocolName {
@@ -31,12 +34,26 @@ pub fn protocol_name(genesis_hash: &[u8], fork_id: Option<&str>) -> ProtocolName
 }

 /// Returns the peers set configuration for the mixnet protocol.
-pub fn peers_set_config(name: ProtocolName, config: &Config) -> NonDefaultSetConfig {
-	let mut set_config = NonDefaultSetConfig::new(name, PACKET_SIZE as u64);
+pub fn peers_set_config(
+	name: ProtocolName,
+	config: &Config,
+) -> (NonDefaultSetConfig, Box<dyn NotificationService>) {
+	let (mut set_config, service) = NonDefaultSetConfig::new(
+		name,
+		Vec::new(),
+		PACKET_SIZE as u64,
+		None,
+		SetConfig {
+			in_peers: 0,
+			out_peers: 0,
+			reserved_nodes: Vec::new(),
+			non_reserved_mode: NonReservedPeerMode::Deny,
+		},
+	);
 	if config.substrate.num_gateway_slots != 0 {
 		// out_peers is always 0; we are only interested in connecting to mixnodes, which we do by
 		// setting them as reserved nodes
 		set_config.allow_non_reserved(config.substrate.num_gateway_slots, 0);
 	}
-	set_config
+	(set_config, service)
 }
@@ -29,11 +29,12 @@ use super::{
 	request::{extrinsic_delay, Request, SUBMIT_EXTRINSIC},
 	sync_with_runtime::sync_with_runtime,
 };
+use bytes::Bytes;
 use codec::{Decode, DecodeAll, Encode};
 use futures::{
 	future::{pending, Either},
 	stream::FuturesUnordered,
-	StreamExt,
+	FutureExt, StreamExt,
 };
 use log::{debug, error, trace, warn};
 use mixnet::{
@@ -43,8 +44,8 @@ use mixnet::{
 };
 use sc_client_api::{BlockchainEvents, HeaderBackend};
 use sc_network::{
-	Event::{NotificationStreamClosed, NotificationStreamOpened, NotificationsReceived},
-	NetworkEventStream, NetworkNotification, NetworkPeers, NetworkStateInfo, ProtocolName,
+	service::traits::{NotificationEvent, ValidationResult},
+	NetworkNotification, NetworkPeers, NetworkStateInfo, NotificationService, ProtocolName,
 };
 use sc_transaction_pool_api::{
 	LocalTransactionPool, OffchainTransactionPoolFactory, TransactionPool,
@@ -154,12 +155,13 @@ pub async fn run<B, C, S, N, P>(
 	protocol_name: ProtocolName,
 	transaction_pool: Arc<P>,
 	keystore: Option<KeystorePtr>,
+	mut notification_service: Box<dyn NotificationService>,
 ) where
 	B: Block,
 	C: BlockchainEvents<B> + ProvideRuntimeApi<B> + HeaderBackend<B>,
 	C::Api: MixnetApi<B>,
 	S: SyncOracle,
-	N: NetworkStateInfo + NetworkEventStream + NetworkNotification + NetworkPeers,
+	N: NetworkStateInfo + NetworkNotification + NetworkPeers,
 	P: TransactionPool<Block = B> + LocalTransactionPool<Block = B> + 'static,
 {
 	let local_peer_id = network.local_peer_id();
@@ -189,7 +191,6 @@ pub async fn run<B, C, S, N, P>(
 	} else {
 		None
 	};
-	let mut network_events = network.event_stream("mixnet").fuse();
 	let mut next_forward_packet_delay = MaybeInfDelay::new(None);
 	let mut next_authored_packet_delay = MaybeInfDelay::new(None);
 	let mut ready_peers = FuturesUnordered::new();
@@ -248,33 +249,36 @@ pub async fn run<B, C, S, N, P>(
 				}
 			}

-			event = network_events.select_next_some() => match event {
-				NotificationStreamOpened { remote, protocol, .. }
-					if protocol == protocol_name => packet_dispatcher.add_peer(&remote),
-				NotificationStreamClosed { remote, protocol }
-					if protocol == protocol_name => packet_dispatcher.remove_peer(&remote),
-				NotificationsReceived { remote, messages } => {
-					for message in messages {
-						if message.0 == protocol_name {
-							match message.1.as_ref().try_into() {
-								Ok(packet) => handle_packet(packet,
-									&mut mixnet, &mut request_manager, &mut reply_manager,
-									&mut extrinsic_queue, &config.substrate),
-								Err(_) => debug!(target: LOG_TARGET,
-									"Dropped incorrectly sized packet ({} bytes) from {remote}",
-									message.1.len(),
-								),
-							}
-						}
+			event = notification_service.next_event().fuse() => match event {
+				None => todo!(),
+				Some(NotificationEvent::ValidateInboundSubstream { result_tx, .. }) => {
+					let _ = result_tx.send(ValidationResult::Accept);
+				},
+				Some(NotificationEvent::NotificationStreamOpened { peer, .. }) => {
+					packet_dispatcher.add_peer(&peer);
+				},
+				Some(NotificationEvent::NotificationStreamClosed { peer }) => {
+					packet_dispatcher.remove_peer(&peer);
+				},
+				Some(NotificationEvent::NotificationReceived { peer, notification }) => {
+					let notification: Bytes = notification.into();
+
+					match notification.as_ref().try_into() {
+						Ok(packet) => handle_packet(packet,
+							&mut mixnet, &mut request_manager, &mut reply_manager,
+							&mut extrinsic_queue, &config.substrate),
+						Err(_) => debug!(target: LOG_TARGET,
+							"Dropped incorrectly sized packet ({} bytes) from {peer}",
+							notification.len(),
+						),
 					}
-				}
-				_ => ()
+				},
 			},

 			_ = next_forward_packet_delay => {
 				if let Some(packet) = mixnet.pop_next_forward_packet() {
 					if let Some(ready_peer) = packet_dispatcher.dispatch(packet) {
-						if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) {
+						if let Some(fut) = ready_peer.send_packet(&notification_service) {
 							ready_peers.push(fut);
 						}
 					}
@@ -288,7 +292,7 @@ pub async fn run<B, C, S, N, P>(
 			_ = next_authored_packet_delay => {
 				if let Some(packet) = mixnet.pop_next_authored_packet(&packet_dispatcher) {
 					if let Some(ready_peer) = packet_dispatcher.dispatch(packet) {
-						if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) {
+						if let Some(fut) = ready_peer.send_packet(&notification_service) {
 							ready_peers.push(fut);
 						}
 					}
@@ -297,7 +301,7 @@ pub async fn run<B, C, S, N, P>(

 			ready_peer = ready_peers.select_next_some() => {
 				if let Some(ready_peer) = ready_peer {
-					if let Some(fut) = ready_peer.send_packet(&*network, protocol_name.clone()) {
+					if let Some(fut) = ready_peer.send_packet(&notification_service) {
 						ready_peers.push(fut);
 					}
 				}
@@ -196,6 +196,7 @@ where
 #[cfg(test)]
 mod tests {
 	use super::*;
+	use multiaddr::multiaddr;

 	#[test]
 	fn fixup_empty_external_addresses() {