Get rid of Peerset compatibility layer (#14337)

* Move bootnodes from individual `SetConfig`s to `PeersetConfig`

* Move `SetId` & `SetConfig` from `peerset` to `protocol_controller`

* Remove unused `DropReason`

* Move `Message` & `IncomingIndex` from `peerset` to `protocol_controller`

* Restore running fuzz test

* Get rid of `Peerset` in `fuzz` test

* Spawn runners instead of manual polling in `fuzz` test

* Migrate `Protocol` from `Peerset` to `PeerStore` & `ProtocolController`

* Migrate `NetworkService` from `Peerset` to `PeerStore` & `ProtocolController`

* Migrate `Notifications` from `Peerset` to `ProtocolController`s

* Migrate `Notifications` tests from `Peerset` to `ProtocolController`

* Fix compilation of `NetworkService` & `Protocol`

* Fix borrowing issues in `Notifications`

* Migrate `RequestResponse`from `Peerset` to `PeerStore`

* rustfmt

* Migrate request-response tests from `Peerset` to `PeerStore`

* Migrate `reconnect_after_disconnect` test to `PeerStore` & `ProtocolController`

* Fix `Notifications` tests

* Remove `Peerset` completely

* Fix bug with counting sync peers in `Protocol`

* Eliminate indirect calls to `PeerStore` via `Protocol`

* Eliminate indirect calls to `ProtocolController` via `Protocol`

* Handle `Err` outcome from `remove_peers_from_reserved_set`

* Add note about disconnecting sync peers in `Protocol`

* minor: remove unneeded `clone()`

* minor: extra comma removed

* minor: use `Stream` API of `from_protocol_controllers` channel

* minor: remove TODO

* minor: replace `.map().flatten()` with `.flat_map()`

* minor: update `ProtocolController` docs

* rustfmt

* Apply suggestions from code review

Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com>

* Extract `MockPeerStore` to `mock.rs`

* Move `PeerStore` initialization to `build_network`

* minor: remove unused import

* minor: clarify error message

* Convert `syncs_header_only_forks` test into single-threaded

---------

Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com>

substrate/client/network/src/protocol.rs

@@ -17,10 +17,10 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{
-	config::{self, NonReservedPeerMode},
-	error,
+	config, error,
+	peer_store::{PeerStoreHandle, PeerStoreProvider},
+	protocol_controller::{self, SetId},
 	types::ProtocolName,
-	ReputationChange,
 };
 
 use bytes::Bytes;
@@ -37,7 +37,7 @@ use libp2p::{
 use log::{debug, error, warn};
 
 use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake};
-use sc_utils::mpsc::TracingUnboundedSender;
+use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender};
 use sp_runtime::traits::Block as BlockT;
 
 use std::{
@@ -62,10 +62,7 @@ pub mod message;
 pub(crate) const BLOCK_ANNOUNCES_TRANSACTIONS_SUBSTREAM_SIZE: u64 = 16 * 1024 * 1024;
 
 /// Identifier of the peerset for the block announces protocol.
-const HARDCODED_PEERSETS_SYNC: crate::peerset::SetId = crate::peerset::SetId::from(0);
-/// Number of hardcoded peersets (the constants right above). Any set whose identifier is equal or
-/// superior to this value corresponds to a user-defined protocol.
-const NUM_HARDCODED_PEERSETS: usize = 1;
+const HARDCODED_PEERSETS_SYNC: SetId = SetId::from(0);
 
 mod rep {
 	use crate::ReputationChange as Rep;
@@ -79,7 +76,7 @@ type PendingSyncSubstreamValidation =
 // Lock must always be taken in order declared here.
 pub struct Protocol<B: BlockT> {
 	/// Used to report reputation changes.
-	peerset_handle: crate::peerset::PeersetHandle,
+	peer_store_handle: PeerStoreHandle,
 	/// Handles opening the unique substream and sending and receiving raw messages.
 	behaviour: Notifications,
 	/// List of notifications protocols that have been registered.
@@ -90,8 +87,8 @@ pub struct Protocol<B: BlockT> {
 	/// event to the outer layers, we also shouldn't propagate this "substream closed" event. To
 	/// solve this, an entry is added to this map whenever an invalid handshake is received.
 	/// Entries are removed when the corresponding "substream closed" is later received.
-	bad_handshake_substreams: HashSet<(PeerId, crate::peerset::SetId)>,
-	/// Connected peers.
+	bad_handshake_substreams: HashSet<(PeerId, SetId)>,
+	/// Connected peers on sync protocol.
 	peers: HashMap<PeerId, Roles>,
 	sync_substream_validations: FuturesUnordered<PendingSyncSubstreamValidation>,
 	tx: TracingUnboundedSender<crate::event::SyncEvent<B>>,
@@ -102,66 +99,17 @@ impl<B: BlockT> Protocol<B> {
 	/// Create a new instance.
 	pub fn new(
 		roles: Roles,
-		network_config: &config::NetworkConfiguration,
 		notification_protocols: Vec<config::NonDefaultSetConfig>,
 		block_announces_protocol: config::NonDefaultSetConfig,
+		peer_store_handle: PeerStoreHandle,
+		protocol_controller_handles: Vec<protocol_controller::ProtocolHandle>,
+		from_protocol_controllers: TracingUnboundedReceiver<protocol_controller::Message>,
 		tx: TracingUnboundedSender<crate::event::SyncEvent<B>>,
-	) -> error::Result<(Self, crate::peerset::PeersetHandle, Vec<(PeerId, Multiaddr)>)> {
-		let mut known_addresses = Vec::new();
-
-		let (peerset, peerset_handle) = {
-			let mut sets =
-				Vec::with_capacity(NUM_HARDCODED_PEERSETS + notification_protocols.len());
-
-			let mut default_sets_reserved = HashSet::new();
-			for reserved in network_config.default_peers_set.reserved_nodes.iter() {
-				default_sets_reserved.insert(reserved.peer_id);
-
-				if !reserved.multiaddr.is_empty() {
-					known_addresses.push((reserved.peer_id, reserved.multiaddr.clone()));
-				}
-			}
-
-			let mut bootnodes = Vec::with_capacity(network_config.boot_nodes.len());
-			for bootnode in network_config.boot_nodes.iter() {
-				bootnodes.push(bootnode.peer_id);
-			}
-
-			// Set number 0 is used for block announces.
-			sets.push(crate::peerset::SetConfig {
-				in_peers: network_config.default_peers_set.in_peers,
-				out_peers: network_config.default_peers_set.out_peers,
-				bootnodes,
-				reserved_nodes: default_sets_reserved.clone(),
-				reserved_only: network_config.default_peers_set.non_reserved_mode ==
-					NonReservedPeerMode::Deny,
-			});
-
-			for set_cfg in &notification_protocols {
-				let mut reserved_nodes = HashSet::new();
-				for reserved in set_cfg.set_config.reserved_nodes.iter() {
-					reserved_nodes.insert(reserved.peer_id);
-					known_addresses.push((reserved.peer_id, reserved.multiaddr.clone()));
-				}
-
-				let reserved_only =
-					set_cfg.set_config.non_reserved_mode == NonReservedPeerMode::Deny;
-
-				sets.push(crate::peerset::SetConfig {
-					in_peers: set_cfg.set_config.in_peers,
-					out_peers: set_cfg.set_config.out_peers,
-					bootnodes: Vec::new(),
-					reserved_nodes,
-					reserved_only,
-				});
-			}
-
-			crate::peerset::Peerset::from_config(crate::peerset::PeersetConfig { sets })
-		};
-
+	) -> error::Result<Self> {
 		let behaviour = {
 			Notifications::new(
-				peerset,
+				protocol_controller_handles,
+				from_protocol_controllers,
 				// NOTE: Block announcement protocol is still very much hardcoded into `Protocol`.
 				// 	This protocol must be the first notification protocol given to
 				// `Notifications`
@@ -181,7 +129,7 @@ impl<B: BlockT> Protocol<B> {
 		};
 
 		let protocol = Self {
-			peerset_handle: peerset_handle.clone(),
+			peer_store_handle,
 			behaviour,
 			notification_protocols: iter::once(block_announces_protocol.notifications_protocol)
 				.chain(notification_protocols.iter().map(|s| s.notifications_protocol.clone()))
@@ -194,7 +142,7 @@ impl<B: BlockT> Protocol<B> {
 			_marker: Default::default(),
 		};
 
-		Ok((protocol, peerset_handle, known_addresses))
+		Ok(protocol)
 	}
 
 	/// Returns the list of all the peers we have an open channel to.
@@ -202,42 +150,28 @@ impl<B: BlockT> Protocol<B> {
 		self.behaviour.open_peers()
 	}
 
-	/// Returns the number of discovered nodes that we keep in memory.
-	pub fn num_discovered_peers(&self) -> usize {
-		self.behaviour.num_discovered_peers()
-	}
-
 	/// Disconnects the given peer if we are connected to it.
 	pub fn disconnect_peer(&mut self, peer_id: &PeerId, protocol_name: ProtocolName) {
 		if let Some(position) = self.notification_protocols.iter().position(|p| *p == protocol_name)
 		{
-			self.behaviour.disconnect_peer(peer_id, crate::peerset::SetId::from(position));
-			self.peers.remove(peer_id);
+			// Note: no need to remove a peer from `self.peers` if we are dealing with sync
+			// protocol, because it will be done when handling
+			// `NotificationsOut::CustomProtocolClosed`.
+			self.behaviour.disconnect_peer(peer_id, SetId::from(position));
 		} else {
 			warn!(target: "sub-libp2p", "disconnect_peer() with invalid protocol name")
 		}
 	}
 
-	/// Returns the state of the peerset manager, for debugging purposes.
-	pub fn peerset_debug_info(&mut self) -> serde_json::Value {
-		self.behaviour.peerset_debug_info()
-	}
-
-	/// Returns the number of peers we're connected to.
+	/// Returns the number of peers we're connected to on sync protocol.
 	pub fn num_connected_peers(&self) -> usize {
 		self.peers.len()
 	}
 
-	/// Adjusts the reputation of a node.
-	pub fn report_peer(&self, who: PeerId, reputation: ReputationChange) {
-		self.peerset_handle.report_peer(who, reputation)
-	}
-
 	/// Set handshake for the notification protocol.
 	pub fn set_notification_handshake(&mut self, protocol: ProtocolName, handshake: Vec<u8>) {
 		if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) {
-			self.behaviour
-				.set_notif_protocol_handshake(crate::peerset::SetId::from(index), handshake);
+			self.behaviour.set_notif_protocol_handshake(SetId::from(index), handshake);
 		} else {
 			error!(
 				target: "sub-libp2p",
@@ -246,81 +180,6 @@ impl<B: BlockT> Protocol<B> {
 			);
 		}
 	}
-
-	/// Set whether the syncing peers set is in reserved-only mode.
-	pub fn set_reserved_only(&self, reserved_only: bool) {
-		self.peerset_handle.set_reserved_only(HARDCODED_PEERSETS_SYNC, reserved_only);
-	}
-
-	/// Removes a `PeerId` from the list of reserved peers for syncing purposes.
-	pub fn remove_reserved_peer(&self, peer: PeerId) {
-		self.peerset_handle.remove_reserved_peer(HARDCODED_PEERSETS_SYNC, peer);
-	}
-
-	/// Returns the list of reserved peers.
-	pub fn reserved_peers(&self, pending_response: oneshot::Sender<Vec<PeerId>>) {
-		self.behaviour.reserved_peers(HARDCODED_PEERSETS_SYNC, pending_response);
-	}
-
-	/// Adds a `PeerId` to the list of reserved peers for syncing purposes.
-	pub fn add_reserved_peer(&self, peer: PeerId) {
-		self.peerset_handle.add_reserved_peer(HARDCODED_PEERSETS_SYNC, peer);
-	}
-
-	/// Sets the list of reserved peers for syncing purposes.
-	pub fn set_reserved_peers(&self, peers: HashSet<PeerId>) {
-		self.peerset_handle.set_reserved_peers(HARDCODED_PEERSETS_SYNC, peers);
-	}
-
-	/// Sets the list of reserved peers for the given protocol/peerset.
-	pub fn set_reserved_peerset_peers(&self, protocol: ProtocolName, peers: HashSet<PeerId>) {
-		if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) {
-			self.peerset_handle
-				.set_reserved_peers(crate::peerset::SetId::from(index), peers);
-		} else {
-			error!(
-				target: "sub-libp2p",
-				"set_reserved_peerset_peers with unknown protocol: {}",
-				protocol
-			);
-		}
-	}
-
-	/// Removes a `PeerId` from the list of reserved peers.
-	pub fn remove_set_reserved_peer(&self, protocol: ProtocolName, peer: PeerId) {
-		if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) {
-			self.peerset_handle
-				.remove_reserved_peer(crate::peerset::SetId::from(index), peer);
-		} else {
-			error!(
-				target: "sub-libp2p",
-				"remove_set_reserved_peer with unknown protocol: {}",
-				protocol
-			);
-		}
-	}
-
-	/// Adds a `PeerId` to the list of reserved peers.
-	pub fn add_set_reserved_peer(&self, protocol: ProtocolName, peer: PeerId) {
-		if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) {
-			self.peerset_handle.add_reserved_peer(crate::peerset::SetId::from(index), peer);
-		} else {
-			error!(
-				target: "sub-libp2p",
-				"add_set_reserved_peer with unknown protocol: {}",
-				protocol
-			);
-		}
-	}
-
-	/// Notify the protocol that we have learned about the existence of some peer.
-	///
-	/// Can be called multiple times with the same `PeerId`.
-	pub fn add_known_peer(&mut self, peer_id: PeerId) {
-		// TODO: get rid of this function and call `Peerset`/`PeerStore` directly
-		// from `NetworkWorker`.
-		self.peerset_handle.add_known_peer(peer_id);
-	}
 }
 
 /// Outcome of an incoming custom message.
@@ -507,7 +366,7 @@ impl<B: BlockT> NetworkBehaviour for Protocol<B> {
 								peer_id,
 								msg,
 							);
-							self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE);
+							self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE);
 							CustomMessageOutcome::None
 						},
 						Err(err) => {
@@ -549,7 +408,7 @@ impl<B: BlockT> NetworkBehaviour for Protocol<B> {
 										err,
 										err2,
 									);
-									self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE);
+									self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE);
 									CustomMessageOutcome::None
 								},
 							}
@@ -586,8 +445,7 @@ impl<B: BlockT> NetworkBehaviour for Protocol<B> {
 							debug!(target: "sync", "Failed to parse remote handshake: {}", err);
 							self.bad_handshake_substreams.insert((peer_id, set_id));
 							self.behaviour.disconnect_peer(&peer_id, set_id);
-							self.peerset_handle.report_peer(peer_id, rep::BAD_MESSAGE);
-							self.peers.remove(&peer_id);
+							self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE);
 							CustomMessageOutcome::None
 						},
 					}