Rework the event system of sc-network (#1370)

This commit introduces a new concept called `NotificationService` which
allows Polkadot protocols to communicate with the underlying
notification protocol implementation directly, without routing events
through `NetworkWorker`. This implies that each protocol has its own
service which it uses to communicate with remote peers and that each
`NotificationService` is unique with respect to the underlying
notification protocol, meaning `NotificationService` for the transaction
protocol can only be used to send and receive transaction-related
notifications.

The `NotificationService` concept introduces two additional benefits:
  * allow protocols to start using custom handshakes
  * allow protocols to accept/reject inbound peers

Previously the validation of inbound connections was solely the
responsibility of `ProtocolController`. This caused issues with light
peers and `SyncingEngine` as `ProtocolController` would accept more
peers than `SyncingEngine` could accept which caused peers to have
differing views of their own states. `SyncingEngine` would reject excess
peers but these rejections were not properly communicated to those peers
causing them to assume that they were accepted.

With `NotificationService`, the local handshake is not sent to remote
peer if peer is rejected which allows it to detect that it was rejected.

This commit also deprecates the use of `NetworkEventStream` for all
notification-related events and going forward only DHT events are
provided through `NetworkEventStream`. If protocols wish to follow each
other's events, they must introduce additional abtractions, as is done
for GRANDPA and transactions protocols by following the syncing protocol
through `SyncEventStream`.

Fixes https://github.com/paritytech/polkadot-sdk/issues/512
Fixes https://github.com/paritytech/polkadot-sdk/issues/514
Fixes https://github.com/paritytech/polkadot-sdk/issues/515
Fixes https://github.com/paritytech/polkadot-sdk/issues/554
Fixes https://github.com/paritytech/polkadot-sdk/issues/556

---
These changes are transferred from
https://github.com/paritytech/substrate/pull/14197 but there are no
functional changes compared to that PR

---------

Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

substrate/client/network/src/service/metrics.rs

@@ -61,9 +61,6 @@ pub struct Metrics {
 	pub kbuckets_num_nodes: GaugeVec<U64>,
 	pub listeners_local_addresses: Gauge<U64>,
 	pub listeners_errors_total: Counter<U64>,
-	pub notifications_sizes: HistogramVec,
-	pub notifications_streams_closed_total: CounterVec<U64>,
-	pub notifications_streams_opened_total: CounterVec<U64>,
 	pub peerset_num_discovered: Gauge<U64>,
 	pub pending_connections: Gauge<U64>,
 	pub pending_connections_errors_total: CounterVec<U64>,
@@ -153,31 +150,6 @@ impl Metrics {
 				"substrate_sub_libp2p_listeners_errors_total",
 				"Total number of non-fatal errors reported by a listener"
 			)?, registry)?,
-			notifications_sizes: prometheus::register(HistogramVec::new(
-				HistogramOpts {
-					common_opts: Opts::new(
-						"substrate_sub_libp2p_notifications_sizes",
-						"Sizes of the notifications send to and received from all nodes"
-					),
-					buckets: prometheus::exponential_buckets(64.0, 4.0, 8)
-						.expect("parameters are always valid values; qed"),
-				},
-				&["direction", "protocol"]
-			)?, registry)?,
-			notifications_streams_closed_total: prometheus::register(CounterVec::new(
-				Opts::new(
-					"substrate_sub_libp2p_notifications_streams_closed_total",
-					"Total number of notification substreams that have been closed"
-				),
-				&["protocol"]
-			)?, registry)?,
-			notifications_streams_opened_total: prometheus::register(CounterVec::new(
-				Opts::new(
-					"substrate_sub_libp2p_notifications_streams_opened_total",
-					"Total number of notification substreams that have been opened"
-				),
-				&["protocol"]
-			)?, registry)?,
 			peerset_num_discovered: prometheus::register(Gauge::new(
 				"substrate_sub_libp2p_peerset_num_discovered",
 				"Number of nodes stored in the peerset manager",

substrate/client/network/src/service/signature.rs

@@ -18,6 +18,8 @@
 //
 // If you read this, you are very thorough, congratulations.
 
+//! Signature-related code
+
 use libp2p::{
 	identity::{Keypair, PublicKey},
 	PeerId,

substrate/client/network/src/service/traits.rs

@@ -18,8 +18,11 @@
 //
 // If you read this, you are very thorough, congratulations.
 
+//! Traits defined by `sc-network`.
+
 use crate::{
 	config::MultiaddrWithPeerId,
+	error,
 	event::Event,
 	request_responses::{IfDisconnected, RequestFailure},
 	service::signature::Signature,
@@ -30,7 +33,9 @@ use crate::{
 use futures::{channel::oneshot, Stream};
 use libp2p::{Multiaddr, PeerId};
 
-use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc};
+use sc_network_common::role::ObservedRole;
+
+use std::{collections::HashSet, fmt::Debug, future::Future, pin::Pin, sync::Arc};
 
 pub use libp2p::{identity::SigningError, kad::record::Key as KademliaKey};
 
@@ -221,6 +226,14 @@ pub trait NetworkPeers {
 
 	/// Returns the number of peers in the sync peer set we're connected to.
 	fn sync_num_connected(&self) -> usize;
+
+	/// Attempt to get peer role.
+	///
+	/// Right now the peer role is decoded from the received handshake for all protocols
+	/// (`/block-announces/1` has other information as well). If the handshake cannot be
+	/// decoded into a role, the role queried from `PeerStore` and if the role is not stored
+	/// there either, `None` is returned and the peer should be discarded.
+	fn peer_role(&self, peer_id: PeerId, handshake: Vec<u8>) -> Option<ObservedRole>;
 }
 
 // Manual implementation to avoid extra boxing here
@@ -296,6 +309,10 @@ where
 	fn sync_num_connected(&self) -> usize {
 		T::sync_num_connected(self)
 	}
+
+	fn peer_role(&self, peer_id: PeerId, handshake: Vec<u8>) -> Option<ObservedRole> {
+		T::peer_role(self, peer_id, handshake)
+	}
 }
 
 /// Provides access to network-level event stream.
@@ -611,3 +628,189 @@ where
 		T::new_best_block_imported(self, hash, number)
 	}
 }
+
+/// Substream acceptance result.
+#[derive(Debug, PartialEq, Eq)]
+pub enum ValidationResult {
+	/// Accept inbound substream.
+	Accept,
+
+	/// Reject inbound substream.
+	Reject,
+}
+
+/// Substream direction.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum Direction {
+	/// Substream opened by the remote node.
+	Inbound,
+
+	/// Substream opened by the local node.
+	Outbound,
+}
+
+impl Direction {
+	/// Is the direction inbound.
+	pub fn is_inbound(&self) -> bool {
+		std::matches!(self, Direction::Inbound)
+	}
+}
+
+/// Events received by the protocol from `Notifications`.
+#[derive(Debug)]
+pub enum NotificationEvent {
+	/// Validate inbound substream.
+	ValidateInboundSubstream {
+		/// Peer ID.
+		peer: PeerId,
+
+		/// Received handshake.
+		handshake: Vec<u8>,
+
+		/// `oneshot::Sender` for sending validation result back to `Notifications`
+		result_tx: tokio::sync::oneshot::Sender<ValidationResult>,
+	},
+
+	/// Remote identified by `PeerId` opened a substream and sent `Handshake`.
+	/// Validate `Handshake` and report status (accept/reject) to `Notifications`.
+	NotificationStreamOpened {
+		/// Peer ID.
+		peer: PeerId,
+
+		/// Is the substream inbound or outbound.
+		direction: Direction,
+
+		/// Received handshake.
+		handshake: Vec<u8>,
+
+		/// Negotiated fallback.
+		negotiated_fallback: Option<ProtocolName>,
+	},
+
+	/// Substream was closed.
+	NotificationStreamClosed {
+		/// Peer Id.
+		peer: PeerId,
+	},
+
+	/// Notification was received from the substream.
+	NotificationReceived {
+		/// Peer ID.
+		peer: PeerId,
+
+		/// Received notification.
+		notification: Vec<u8>,
+	},
+}
+
+/// Notification service
+///
+/// Defines behaviors that both the protocol implementations and `Notifications` can expect from
+/// each other.
+///
+/// `Notifications` can send two different kinds of information to protocol:
+///  * substream-related information
+///  * notification-related information
+///
+/// When an unvalidated, inbound substream is received by `Notifications`, it sends the inbound
+/// stream information (peer ID, handshake) to protocol for validation. Protocol must then verify
+/// that the handshake is valid (and in the future that it has a slot it can allocate for the peer)
+/// and then report back the `ValidationResult` which is either `Accept` or `Reject`.
+///
+/// After the validation result has been received by `Notifications`, it prepares the
+/// substream for communication by initializing the necessary sinks and emits
+/// `NotificationStreamOpened` which informs the protocol that the remote peer is ready to receive
+/// notifications.
+///
+/// Two different flavors of sending options are provided:
+///  * synchronous sending ([`NotificationService::send_sync_notification()`])
+///  * asynchronous sending ([`NotificationService::send_async_notification()`])
+///
+/// The former is used by the protocols not ready to exercise backpressure and the latter by the
+/// protocols that can do it.
+///
+/// Both local and remote peer can close the substream at any time. Local peer can do so by calling
+/// [`NotificationService::close_substream()`] which instructs `Notifications` to close the
+/// substream. Remote closing the substream is indicated to the local peer by receiving
+/// [`NotificationEvent::NotificationStreamClosed`] event.
+///
+/// In case the protocol must update its handshake while it's operating (such as updating the best
+/// block information), it can do so by calling [`NotificationService::set_handshake()`]
+/// which instructs `Notifications` to update the handshake it stored during protocol
+/// initialization.
+///
+/// All peer events are multiplexed on the same incoming event stream from `Notifications` and thus
+/// each event carries a `PeerId` so the protocol knows whose information to update when receiving
+/// an event.
+#[async_trait::async_trait]
+pub trait NotificationService: Debug + Send {
+	/// Instruct `Notifications` to open a new substream for `peer`.
+	///
+	/// `dial_if_disconnected` informs `Notifications` whether to dial
+	// the peer if there is currently no active connection to it.
+	//
+	// NOTE: not offered by the current implementation
+	async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()>;
+
+	/// Instruct `Notifications` to close substream for `peer`.
+	//
+	// NOTE: not offered by the current implementation
+	async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>;
+
+	/// Send synchronous `notification` to `peer`.
+	fn send_sync_notification(&self, peer: &PeerId, notification: Vec<u8>);
+
+	/// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure.
+	///
+	/// Returns an error if the peer doesn't exist.
+	async fn send_async_notification(
+		&self,
+		peer: &PeerId,
+		notification: Vec<u8>,
+	) -> Result<(), error::Error>;
+
+	/// Set handshake for the notification protocol replacing the old handshake.
+	async fn set_handshake(&mut self, handshake: Vec<u8>) -> Result<(), ()>;
+
+	/// Non-blocking variant of `set_handshake()` that attempts to update the handshake
+	/// and returns an error if the channel is blocked.
+	///
+	/// Technically the function can return an error if the channel to `Notifications` is closed
+	/// but that doesn't happen under normal operation.
+	fn try_set_handshake(&mut self, handshake: Vec<u8>) -> Result<(), ()>;
+
+	/// Get next event from the `Notifications` event stream.
+	async fn next_event(&mut self) -> Option<NotificationEvent>;
+
+	/// Make a copy of the object so it can be shared between protocol components
+	/// who wish to have access to the same underlying notification protocol.
+	fn clone(&mut self) -> Result<Box<dyn NotificationService>, ()>;
+
+	/// Get protocol name of the `NotificationService`.
+	fn protocol(&self) -> &ProtocolName;
+
+	/// Get message sink of the peer.
+	fn message_sink(&self, peer: &PeerId) -> Option<Box<dyn MessageSink>>;
+}
+
+/// Message sink for peers.
+///
+/// If protocol cannot use [`NotificationService`] to send notifications to peers and requires,
+/// e.g., notifications to be sent in another task, the protocol may acquire a [`MessageSink`]
+/// object for each peer by calling [`NotificationService::message_sink()`]. Calling this
+/// function returns an object which allows the protocol to send notifications to the remote peer.
+///
+/// Use of this API is discouraged as it's not as performant as sending notifications through
+/// [`NotificationService`] due to synchronization required to keep the underlying notification
+/// sink up to date with possible sink replacement events.
+#[async_trait::async_trait]
+pub trait MessageSink: Send + Sync {
+	/// Send synchronous `notification` to the peer associated with this [`MessageSink`].
+	fn send_sync_notification(&self, notification: Vec<u8>);
+
+	/// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`],
+	/// allowing sender to exercise backpressure.
+	///
+	/// Returns an error if the peer does not exist.
+	async fn send_async_notification(&self, notification: Vec<u8>) -> Result<(), error::Error>;
+}