Rework priority groups, take 2 (#7700)

* Rework priority groups

* Broken tests fix

* Fix warning causing CI to fail

* [Hack] Try restore backwards-compatibility

* Fix peerset bug

* Doc fixes and clean up

* Error on state mismatch

* Try debug CI

* CI debugging

* [CI debug] Can I please see this line

* Revert "[CI debug] Can I please see this line"

This reverts commit 4b7cf7c1511f579cd818b21d46bd11642dfac5cb.

* Revert "CI debugging"

This reverts commit 9011f1f564b860386dc7dd6ffa9fc34ea7107623.

* Fix error! which isn't actually an error

* Fix Ok() returned when actually Err()

* Tweaks and fixes

* Fix build

* Peerset bugfix

* [Debug] Try outbound GrandPa slots

* Another bugfix

* Revert "[Debug] Try outbound GrandPa slots"

This reverts commit d175b9208c088faad77d9f0ce36ff6f48bd92dd3.

* [Debug] Try outbound GrandPa slots

* Apply suggestions from code review

Co-authored-by: Max Inden <mail@max-inden.de>

* Use consts for hardcoded peersets

* Revert "Try debug CI"

This reverts commit 62c4ad5e79c03d561c714a008022ecac463a597e.

* Renames

* Line widths

* Add doc

Co-authored-by: Max Inden <mail@max-inden.de>

substrate/bin/node-template/node/src/service.rs

@@ -107,7 +107,8 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 			}
 		};
 	}
-	config.network.notifications_protocols.push(sc_finality_grandpa::GRANDPA_PROTOCOL_NAME.into());
+
+	config.network.extra_sets.push(sc_finality_grandpa::grandpa_peers_set_config());
 
 	let (network, network_status_sinks, system_rpc_tx, network_starter) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
@@ -244,7 +245,7 @@ pub fn new_light(mut config: Configuration) -> Result<TaskManager, ServiceError>
 	let (client, backend, keystore_container, mut task_manager, on_demand) =
 		sc_service::new_light_parts::<Block, RuntimeApi, Executor>(&config)?;
 
-	config.network.notifications_protocols.push(sc_finality_grandpa::GRANDPA_PROTOCOL_NAME.into());
+	config.network.extra_sets.push(sc_finality_grandpa::grandpa_peers_set_config());
 
 	let select_chain = sc_consensus::LongestChain::new(backend.clone());
 

substrate/bin/node/cli/src/service.rs

@@ -178,7 +178,7 @@ pub fn new_full_base(
 
 	let shared_voter_state = rpc_setup;
 
-	config.network.notifications_protocols.push(grandpa::GRANDPA_PROTOCOL_NAME.into());
+	config.network.extra_sets.push(grandpa::grandpa_peers_set_config());
 
 	let (network, network_status_sinks, system_rpc_tx, network_starter) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
@@ -346,7 +346,7 @@ pub fn new_light_base(mut config: Configuration) -> Result<(
 	let (client, backend, keystore_container, mut task_manager, on_demand) =
 		sc_service::new_light_parts::<Block, RuntimeApi, Executor>(&config)?;
 
-	config.network.notifications_protocols.push(grandpa::GRANDPA_PROTOCOL_NAME.into());
+	config.network.extra_sets.push(grandpa::grandpa_peers_set_config());
 
 	let select_chain = sc_consensus::LongestChain::new(backend.clone());
 

substrate/client/authority-discovery/src/error.rs

@@ -38,8 +38,6 @@ pub enum Error {
 	CallingRuntime(sp_blockchain::Error),
 	/// Received a dht record with a key that does not match any in-flight awaited keys.
 	ReceivingUnexpectedRecord,
-	/// Failed to set the authority discovery peerset priority group in the peerset module.
-	SettingPeersetPriorityGroup(String),
 	/// Failed to encode a protobuf payload.
 	EncodingProto(prost::EncodeError),
 	/// Failed to decode a protobuf payload.

substrate/client/authority-discovery/src/worker.rs

@@ -57,10 +57,6 @@ pub mod tests;
 
 const LOG_TARGET: &'static str = "sub-authority-discovery";
 
-/// Name of the Substrate peerset priority group for authorities discovered through the authority
-/// discovery module.
-const AUTHORITIES_PRIORITY_GROUP_NAME: &'static str = "authorities";
-
 /// Maximum number of addresses cached per authority. Additional addresses are discarded.
 const MAX_ADDRESSES_PER_AUTHORITY: usize = 10;
 
@@ -115,9 +111,6 @@ pub struct Worker<Client, Network, Block, DhtEventStream> {
 	publish_interval: ExpIncInterval,
 	/// Interval at which to request addresses of authorities, refilling the pending lookups queue.
 	query_interval: ExpIncInterval,
-	/// Interval on which to set the peerset priority group to a new random
-	/// set of addresses.
-	priority_group_set_interval: ExpIncInterval,
 
 	/// Queue of throttled lookups pending to be passed to the network.
 	pending_lookups: Vec<AuthorityId>,
@@ -166,13 +159,6 @@ where
 			Duration::from_secs(2),
 			config.max_query_interval,
 		);
-		let priority_group_set_interval = ExpIncInterval::new(
-			Duration::from_secs(2),
-			// Trade-off between node connection churn and connectivity. Using half of
-			// [`crate::WorkerConfig::max_query_interval`] to update priority group once at the
-			// beginning and once in the middle of each query interval.
-			config.max_query_interval / 2,
-		);
 
 		let addr_cache = AddrCache::new();
 
@@ -196,7 +182,6 @@ where
 			dht_event_rx,
 			publish_interval,
 			query_interval,
-			priority_group_set_interval,
 			pending_lookups: Vec::new(),
 			in_flight_lookups: HashMap::new(),
 			addr_cache,
@@ -226,15 +211,6 @@ where
 				msg = self.from_service.select_next_some() => {
 					self.process_message_from_service(msg);
 				},
-				// Set peerset priority group to a new random set of addresses.
-				_ = self.priority_group_set_interval.next().fuse() => {
-					if let Err(e) = self.set_priority_group().await {
-						error!(
-							target: LOG_TARGET,
-							"Failed to set priority group: {:?}", e,
-						);
-					}
-				},
 				// Publish own addresses.
 				_ = self.publish_interval.next().fuse() => {
 					if let Err(e) = self.publish_ext_addresses().await {
@@ -582,38 +558,6 @@ where
 
 		Ok(intersection)
 	}
-
-	/// Set the peer set 'authority' priority group to a new random set of
-	/// [`Multiaddr`]s.
-	async fn set_priority_group(&self) -> Result<()> {
-		let addresses = self.addr_cache.get_random_subset();
-
-		if addresses.is_empty() {
-			debug!(
-				target: LOG_TARGET,
-				"Got no addresses in cache for peerset priority group.",
-			);
-			return Ok(());
-		}
-
-		if let Some(metrics) = &self.metrics {
-			metrics.priority_group_size.set(addresses.len().try_into().unwrap_or(std::u64::MAX));
-		}
-
-		debug!(
-			target: LOG_TARGET,
-			"Applying priority group {:?} to peerset.", addresses,
-		);
-
-		self.network
-			.set_priority_group(
-				AUTHORITIES_PRIORITY_GROUP_NAME.to_string(),
-				addresses.into_iter().collect(),
-			).await
-			.map_err(Error::SettingPeersetPriorityGroup)?;
-
-		Ok(())
-	}
 }
 
 /// NetworkProvider provides [`Worker`] with all necessary hooks into the
@@ -621,13 +565,6 @@ where
 /// [`sc_network::NetworkService`] directly is necessary to unit test [`Worker`].
 #[async_trait]
 pub trait NetworkProvider: NetworkStateInfo {
-	/// Modify a peerset priority group.
-	async fn set_priority_group(
-		&self,
-		group_id: String,
-		peers: HashSet<libp2p::Multiaddr>,
-	) -> std::result::Result<(), String>;
-
 	/// Start putting a value in the Dht.
 	fn put_value(&self, key: libp2p::kad::record::Key, value: Vec<u8>);
 
@@ -641,13 +578,6 @@ where
 	B: BlockT + 'static,
 	H: ExHashT,
 {
-	async fn set_priority_group(
-		&self,
-		group_id: String,
-		peers: HashSet<libp2p::Multiaddr>,
-	) -> std::result::Result<(), String> {
-		self.set_priority_group(group_id, peers).await
-	}
 	fn put_value(&self, key: libp2p::kad::record::Key, value: Vec<u8>) {
 		self.put_value(key, value)
 	}
@@ -670,7 +600,6 @@ pub(crate) struct Metrics {
 	dht_event_received: CounterVec<U64>,
 	handle_value_found_event_failure: Counter<U64>,
 	known_authorities_count: Gauge<U64>,
-	priority_group_size: Gauge<U64>,
 }
 
 impl Metrics {
@@ -730,13 +659,6 @@ impl Metrics {
 				)?,
 				registry,
 			)?,
-			priority_group_size: register(
-				Gauge::new(
-					"authority_discovery_priority_group_size",
-					"Number of addresses passed to the peer set as a priority group."
-				)?,
-				registry,
-			)?,
 		})
 	}
 }

substrate/client/authority-discovery/src/worker/addr_cache.rs

@@ -17,17 +17,11 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use libp2p::core::multiaddr::{Multiaddr, Protocol};
-use rand::seq::SliceRandom;
 use std::collections::HashMap;
 
 use sp_authority_discovery::AuthorityId;
 use sc_network::PeerId;
 
-/// The maximum number of authority connections initialized through the authority discovery module.
-///
-/// In other words the maximum size of the `authority` peerset priority group.
-const MAX_NUM_AUTHORITY_CONN: usize = 10;
-
 /// Cache for [`AuthorityId`] -> [`Vec<Multiaddr>`] and [`PeerId`] -> [`AuthorityId`] mappings.
 pub(super) struct AddrCache {
 	authority_id_to_addresses: HashMap<AuthorityId, Vec<Multiaddr>>,
@@ -77,30 +71,6 @@ impl AddrCache {
 		self.peer_id_to_authority_id.get(peer_id)
 	}
 
-	/// Returns a single address for a random subset (maximum of [`MAX_NUM_AUTHORITY_CONN`]) of all
-	/// known authorities.
-	pub fn get_random_subset(&self) -> Vec<Multiaddr> {
-		let mut rng = rand::thread_rng();
-
-		let mut addresses = self
-			.authority_id_to_addresses
-			.iter()
-			.filter_map(|(_authority_id, addresses)| {
-				debug_assert!(!addresses.is_empty());
-				addresses
-					.choose(&mut rng)
-			})
-			.collect::<Vec<&Multiaddr>>();
-
-		addresses.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref()));
-		addresses.dedup();
-
-		addresses
-			.choose_multiple(&mut rng, MAX_NUM_AUTHORITY_CONN)
-			.map(|a| (**a).clone())
-			.collect()
-	}
-
 	/// Removes all [`PeerId`]s and [`Multiaddr`]s from the cache that are not related to the given
 	/// [`AuthorityId`]s.
 	pub fn retain_ids(&mut self, authority_ids: &Vec<AuthorityId>) {
@@ -192,11 +162,6 @@ mod tests {
 			cache.insert(second.0.clone(), vec![second.1.clone()]);
 			cache.insert(third.0.clone(), vec![third.1.clone()]);
 
-			let subset = cache.get_random_subset();
-			assert!(
-				subset.contains(&first.1) && subset.contains(&second.1) && subset.contains(&third.1),
-				"Expect initial subset to contain all authorities.",
-			);
 			assert_eq!(
 				Some(&vec![third.1.clone()]),
 				cache.get_addresses_by_authority_id(&third.0),
@@ -210,12 +175,6 @@ mod tests {
 
 			cache.retain_ids(&vec![first.0, second.0]);
 
-			let subset = cache.get_random_subset();
-			assert!(
-				subset.contains(&first.1) || subset.contains(&second.1),
-				"Expected both first and second authority."
-			);
-			assert!(!subset.contains(&third.1), "Did not expect address from third authority");
 			assert_eq!(
 				None, cache.get_addresses_by_authority_id(&third.0),
 				"Expect `get_addresses_by_authority_id` to not return `None` for third authority."

substrate/client/authority-discovery/src/worker/tests.rs

@@ -18,7 +18,7 @@
 
 use crate::worker::schema;
 
-use std::{iter::FromIterator, sync::{Arc, Mutex}, task::Poll};
+use std::{sync::{Arc, Mutex}, task::Poll};
 
 use async_trait::async_trait;
 use futures::channel::mpsc::{self, channel};
@@ -112,10 +112,6 @@ sp_api::mock_impl_runtime_apis! {
 pub enum TestNetworkEvent {
 	GetCalled(kad::record::Key),
 	PutCalled(kad::record::Key, Vec<u8>),
-	SetPriorityGroupCalled {
-		group_id: String,
-		peers: HashSet<Multiaddr>
-	},
 }
 
 pub struct TestNetwork {
@@ -125,7 +121,6 @@ pub struct TestNetwork {
 	// vectors below.
 	pub put_value_call: Arc<Mutex<Vec<(kad::record::Key, Vec<u8>)>>>,
 	pub get_value_call: Arc<Mutex<Vec<kad::record::Key>>>,
-	pub set_priority_group_call: Arc<Mutex<Vec<(String, HashSet<Multiaddr>)>>>,
 	event_sender: mpsc::UnboundedSender<TestNetworkEvent>,
 	event_receiver: Option<mpsc::UnboundedReceiver<TestNetworkEvent>>,
 }
@@ -147,7 +142,6 @@ impl Default for TestNetwork {
 			],
 			put_value_call: Default::default(),
 			get_value_call: Default::default(),
-			set_priority_group_call: Default::default(),
 			event_sender: tx,
 			event_receiver: Some(rx),
 		}
@@ -156,21 +150,6 @@ impl Default for TestNetwork {
 
 #[async_trait]
 impl NetworkProvider for TestNetwork {
-	async fn set_priority_group(
-		&self,
-		group_id: String,
-		peers: HashSet<Multiaddr>,
-	) -> std::result::Result<(), String> {
-		self.set_priority_group_call
-			.lock()
-			.unwrap()
-			.push((group_id.clone(), peers.clone()));
-		self.event_sender.clone().unbounded_send(TestNetworkEvent::SetPriorityGroupCalled {
-			group_id,
-			peers,
-		}).unwrap();
-		Ok(())
-	}
 	fn put_value(&self, key: kad::record::Key, value: Vec<u8>) {
 		self.put_value_call.lock().unwrap().push((key.clone(), value.clone()));
 		self.event_sender.clone().unbounded_send(TestNetworkEvent::PutCalled(key, value)).unwrap();
@@ -296,14 +275,6 @@ fn publish_discover_cycle() {
 	let (_dht_event_tx, dht_event_rx) = channel(1000);
 
 	let network: Arc<TestNetwork> = Arc::new(Default::default());
-	let node_a_multiaddr = {
-		let peer_id = network.local_peer_id();
-		let address = network.external_addresses().pop().unwrap();
-
-		address.with(multiaddr::Protocol::P2p(
-			peer_id.into(),
-		))
-	};
 
 	let key_store = KeyStore::new();
 
@@ -365,19 +336,6 @@ fn publish_discover_cycle() {
 
 		// Make authority discovery handle the event.
 		worker.handle_dht_event(dht_event).await;
-
-		worker.set_priority_group().await.unwrap();
-
-		// Expect authority discovery to set the priority set.
-		assert_eq!(network.set_priority_group_call.lock().unwrap().len(), 1);
-
-		assert_eq!(
-			network.set_priority_group_call.lock().unwrap()[0],
-			(
-				"authorities".to_string(),
-				HashSet::from_iter(vec![node_a_multiaddr.clone()].into_iter())
-			)
-		);
 	}.boxed_local().into());
 
 	pool.run();

substrate/client/cli/src/params/network_params.rs

@@ -18,7 +18,7 @@
 
 use crate::params::node_key_params::NodeKeyParams;
 use sc_network::{
-	config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, TransportConfig},
+	config::{NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig},
 	multiaddr::Protocol,
 };
 use sc_service::{ChainSpec, ChainType, config::{Multiaddr, MultiaddrWithPeerId}};
@@ -150,21 +150,23 @@ impl NetworkParams {
 		NetworkConfiguration {
 			boot_nodes,
 			net_config_path,
-			reserved_nodes: self.reserved_nodes.clone(),
+			default_peers_set: SetConfig {
-			non_reserved_mode: if self.reserved_only {
+				in_peers: self.in_peers,
-				NonReservedPeerMode::Deny
+				out_peers: self.out_peers,
-			} else {
+				reserved_nodes: self.reserved_nodes.clone(),
-				NonReservedPeerMode::Accept
+				non_reserved_mode: if self.reserved_only {
+					NonReservedPeerMode::Deny
+				} else {
+					NonReservedPeerMode::Accept
+				},
 			},
 			listen_addresses,
 			public_addresses,
-			notifications_protocols: Vec::new(),
+			extra_sets: Vec::new(),
 			request_response_protocols: Vec::new(),
 			node_key,
 			node_name: node_name.to_string(),
 			client_version: client_id.to_string(),
-			in_peers: self.in_peers,
-			out_peers: self.out_peers,
 			transport: TransportConfig::Normal {
 				enable_mdns: !is_dev && !self.no_mdns,
 				allow_private_ipv4: !self.no_private_ipv4,

substrate/client/finality-grandpa/src/communication/tests.rs

@@ -58,7 +58,11 @@ impl sc_network_gossip::Network<Block> for TestNetwork {
 		let _ = self.sender.unbounded_send(Event::Report(who, cost_benefit));
 	}
 
-	fn disconnect_peer(&self, _: PeerId) {}
+	fn add_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {}
+
+	fn remove_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {}
+
+	fn disconnect_peer(&self, _: PeerId, _: Cow<'static, str>) {}
 
 	fn write_notification(&self, who: PeerId, _: Cow<'static, str>, message: Vec<u8>) {
 		let _ = self.sender.unbounded_send(Event::WriteNotification(who, message));

substrate/client/finality-grandpa/src/lib.rs

@@ -122,7 +122,6 @@ mod until_imported;
 mod voting_rule;
 
 pub use authorities::{SharedAuthoritySet, AuthoritySet};
-pub use communication::GRANDPA_PROTOCOL_NAME;
 pub use finality_proof::{FinalityProofFragment, FinalityProofProvider, StorageAndProofProvider};
 pub use notification::{GrandpaJustificationSender, GrandpaJustificationStream};
 pub use import::GrandpaBlockImport;
@@ -656,7 +655,7 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
 	///
 	/// It is assumed that this network will feed us Grandpa notifications. When using the
 	/// `sc_network` crate, it is assumed that the Grandpa notifications protocol has been passed
-	/// to the configuration of the networking.
+	/// to the configuration of the networking. See [`grandpa_peers_set_config`].
 	pub network: N,
 	/// If supplied, can be used to hook on telemetry connection established events.
 	pub telemetry_on_connect: Option<TracingUnboundedReceiver<()>>,
@@ -668,6 +667,20 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
 	pub shared_voter_state: SharedVoterState,
 }
 
+/// Returns the configuration value to put in
+/// [`sc_network::config::NetworkConfiguration::extra_sets`].
+pub fn grandpa_peers_set_config() -> sc_network::config::NonDefaultSetConfig {
+	sc_network::config::NonDefaultSetConfig {
+		notifications_protocol: communication::GRANDPA_PROTOCOL_NAME.into(),
+		set_config: sc_network::config::SetConfig {
+			in_peers: 25,
+			out_peers: 25,
+			reserved_nodes: Vec::new(),
+			non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept,
+		},
+	}
+}
+
 /// Run a GRANDPA voter as a task. Provide configuration and a link to a
 /// block import worker that has already been instantiated with `block_import`.
 pub fn run_grandpa_voter<Block: BlockT, BE: 'static, C, N, SC, VR>(

substrate/client/network-gossip/src/bridge.rs

@@ -180,6 +180,12 @@ impl<B: BlockT> Future for GossipEngine<B> {
 				ForwardingState::Idle => {
 					match this.network_event_stream.poll_next_unpin(cx) {
 						Poll::Ready(Some(event)) => match event {
+							Event::SyncConnected { remote } => {
+								this.network.add_set_reserved(remote, this.protocol.clone());
+							}
+							Event::SyncDisconnected { remote } => {
+								this.network.remove_set_reserved(remote, this.protocol.clone());
+							}
 							Event::NotificationStreamOpened { remote, protocol, role } => {
 								if protocol != this.protocol {
 									continue;
@@ -325,10 +331,16 @@ mod tests {
 		fn report_peer(&self, _: PeerId, _: ReputationChange) {
 		}
 
-		fn disconnect_peer(&self, _: PeerId) {
+		fn disconnect_peer(&self, _: PeerId, _: Cow<'static, str>) {
 			unimplemented!();
 		}
 
+		fn add_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {
+		}
+
+		fn remove_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {
+		}
+
 		fn write_notification(&self, _: PeerId, _: Cow<'static, str>, _: Vec<u8>) {
 			unimplemented!();
 		}

substrate/client/network-gossip/src/lib.rs

@@ -40,6 +40,11 @@
 //! - Use the methods of the `GossipEngine` in order to send out messages and receive incoming
 //!   messages.
 //!
+//! The `GossipEngine` will automatically use `Network::add_set_reserved` and
+//! `Network::remove_set_reserved` to maintain a set of peers equal to the set of peers the
+//! node is syncing from. See the documentation of `sc-network` for more explanations about the
+//! concepts of peer sets.
+//!
 //! # What is a validator?
 //!
 //! The primary role of a `Validator` is to process incoming messages from peers, and decide
@@ -61,9 +66,9 @@ pub use self::state_machine::TopicNotification;
 pub use self::validator::{DiscardAll, MessageIntent, Validator, ValidatorContext, ValidationResult};
 
 use futures::prelude::*;
-use sc_network::{Event, ExHashT, NetworkService, PeerId, ReputationChange};
+use sc_network::{multiaddr, Event, ExHashT, NetworkService, PeerId, ReputationChange};
 use sp_runtime::{traits::Block as BlockT};
-use std::{borrow::Cow, pin::Pin, sync::Arc};
+use std::{borrow::Cow, iter, pin::Pin, sync::Arc};
 
 mod bridge;
 mod state_machine;
@@ -77,8 +82,14 @@ pub trait Network<B: BlockT> {
 	/// Adjust the reputation of a node.
 	fn report_peer(&self, peer_id: PeerId, reputation: ReputationChange);
 
+	/// Adds the peer to the set of peers to be connected to with this protocol.
+	fn add_set_reserved(&self, who: PeerId, protocol: Cow<'static, str>);
+
+	/// Removes the peer from the set of peers to be connected to with this protocol.
+	fn remove_set_reserved(&self, who: PeerId, protocol: Cow<'static, str>);
+
 	/// Force-disconnect a peer.
-	fn disconnect_peer(&self, who: PeerId);
+	fn disconnect_peer(&self, who: PeerId, protocol: Cow<'static, str>);
 
 	/// Send a notification to a peer.
 	fn write_notification(&self, who: PeerId, protocol: Cow<'static, str>, message: Vec<u8>);
@@ -99,8 +110,26 @@ impl<B: BlockT, H: ExHashT> Network<B> for Arc<NetworkService<B, H>> {
 		NetworkService::report_peer(self, peer_id, reputation);
 	}
 
-	fn disconnect_peer(&self, who: PeerId) {
+	fn add_set_reserved(&self, who: PeerId, protocol: Cow<'static, str>) {
-		NetworkService::disconnect_peer(self, who)
+		let addr = iter::once(multiaddr::Protocol::P2p(who.into()))
+			.collect::<multiaddr::Multiaddr>();
+		let result = NetworkService::add_to_peers_set(self, protocol, iter::once(addr).collect());
+		if let Err(err) = result {
+			log::error!(target: "gossip", "add_set_reserved failed: {}", err);
+		}
+	}
+
+	fn remove_set_reserved(&self, who: PeerId, protocol: Cow<'static, str>) {
+		let addr = iter::once(multiaddr::Protocol::P2p(who.into()))
+			.collect::<multiaddr::Multiaddr>();
+		let result = NetworkService::remove_from_peers_set(self, protocol, iter::once(addr).collect());
+		if let Err(err) = result {
+			log::error!(target: "gossip", "remove_set_reserved failed: {}", err);
+		}
+	}
+
+	fn disconnect_peer(&self, who: PeerId, protocol: Cow<'static, str>) {
+		NetworkService::disconnect_peer(self, who, protocol)
 	}
 
 	fn write_notification(&self, who: PeerId, protocol: Cow<'static, str>, message: Vec<u8>) {

substrate/client/network-gossip/src/state_machine.rs

@@ -495,10 +495,16 @@ mod tests {
 			self.inner.lock().unwrap().peer_reports.push((peer_id, reputation_change));
 		}
 
-		fn disconnect_peer(&self, _: PeerId) {
+		fn disconnect_peer(&self, _: PeerId, _: Cow<'static, str>) {
 			unimplemented!();
 		}
 
+		fn add_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {
+		}
+
+		fn remove_set_reserved(&self, _: PeerId, _: Cow<'static, str>) {
+		}
+
 		fn write_notification(&self, _: PeerId, _: Cow<'static, str>, _: Vec<u8>) {
 			unimplemented!();
 		}

substrate/client/network/src/behaviour.rs

@@ -24,7 +24,6 @@ use crate::{
 };
 
 use bytes::Bytes;
-use codec::Encode as _;
 use futures::channel::oneshot;
 use libp2p::NetworkBehaviour;
 use libp2p::core::{Multiaddr, PeerId, PublicKey};
@@ -157,6 +156,12 @@ pub enum BehaviourOut<B: BlockT> {
 		messages: Vec<(Cow<'static, str>, Bytes)>,
 	},
 
+	/// Now connected to a new peer for syncing purposes.
+	SyncConnected(PeerId),
+
+	/// No longer connected to a peer for syncing purposes.
+	SyncDisconnected(PeerId),
+
 	/// Events generated by a DHT as a response to get_value or put_value requests as well as the
 	/// request duration.
 	Dht(DhtEvent, Duration),
@@ -242,35 +247,6 @@ impl<B: BlockT, H: ExHashT> Behaviour<B, H> {
 		self.request_responses.send_request(target, protocol, request, pending_response)
 	}
 
-	/// Registers a new notifications protocol.
-	///
-	/// Please call `event_stream` before registering a protocol, otherwise you may miss events
-	/// about the protocol that you have registered.
-	///
-	/// You are very strongly encouraged to call this method very early on. Any connection open
-	/// will retain the protocols that were registered then, and not any new one.
-	pub fn register_notifications_protocol(
-		&mut self,
-		protocol: impl Into<Cow<'static, str>>,
-	) {
-		let protocol = protocol.into();
-
-		// This is the message that we will send to the remote as part of the initial handshake.
-		// At the moment, we force this to be an encoded `Roles`.
-		let handshake_message = Roles::from(&self.role).encode();
-
-		let list = self.substrate.register_notifications_protocol(protocol.clone(), handshake_message);
-		for (remote, roles, notifications_sink) in list {
-			let role = reported_roles_to_observed_role(&self.role, remote, roles);
-			self.events.push_back(BehaviourOut::NotificationStreamOpened {
-				remote: remote.clone(),
-				protocol: protocol.clone(),
-				role,
-				notifications_sink: notifications_sink.clone(),
-			});
-		}
-	}
-
 	/// Returns a shared reference to the user protocol.
 	pub fn user_protocol(&self) -> &Protocol<B, H> {
 		&self.substrate
@@ -343,38 +319,36 @@ Behaviour<B, H> {
 					&target, &self.block_request_protocol_name, buf, pending_response,
 				);
 			},
-			CustomMessageOutcome::NotificationStreamOpened { remote, protocols, roles, notifications_sink } => {
+			CustomMessageOutcome::NotificationStreamOpened { remote, protocol, roles, notifications_sink } => {
 				let role = reported_roles_to_observed_role(&self.role, &remote, roles);
-				for protocol in protocols {
+				self.events.push_back(BehaviourOut::NotificationStreamOpened {
-					self.events.push_back(BehaviourOut::NotificationStreamOpened {
+					remote,
-						remote: remote.clone(),
+					protocol,
-						protocol,
+					role: role.clone(),
-						role: role.clone(),
+					notifications_sink: notifications_sink.clone(),
-						notifications_sink: notifications_sink.clone(),
+				});
-					});
-				}
 			},
-			CustomMessageOutcome::NotificationStreamReplaced { remote, protocols, notifications_sink } =>
+			CustomMessageOutcome::NotificationStreamReplaced { remote, protocol, notifications_sink } =>
-				for protocol in protocols {
+				self.events.push_back(BehaviourOut::NotificationStreamReplaced {
-					self.events.push_back(BehaviourOut::NotificationStreamReplaced {
+					remote,
-						remote: remote.clone(),
+					protocol,
-						protocol,
+					notifications_sink,
-						notifications_sink: notifications_sink.clone(),
+				}),
-					});
+			CustomMessageOutcome::NotificationStreamClosed { remote, protocol } =>
-				},
+				self.events.push_back(BehaviourOut::NotificationStreamClosed {
-			CustomMessageOutcome::NotificationStreamClosed { remote, protocols } =>
+					remote,
-				for protocol in protocols {
+					protocol,
-					self.events.push_back(BehaviourOut::NotificationStreamClosed {
+				}),
-						remote: remote.clone(),
-						protocol,
-					});
-				},
 			CustomMessageOutcome::NotificationsReceived { remote, messages } => {
 				self.events.push_back(BehaviourOut::NotificationsReceived { remote, messages });
 			},
 			CustomMessageOutcome::PeerNewBest(peer_id, number) => {
 				self.light_client_handler.update_best_block(&peer_id, number);
 			}
+			CustomMessageOutcome::SyncConnected(peer_id) =>
+				self.events.push_back(BehaviourOut::SyncConnected(peer_id)),
+			CustomMessageOutcome::SyncDisconnected(peer_id) =>
+				self.events.push_back(BehaviourOut::SyncDisconnected(peer_id)),
 			CustomMessageOutcome::None => {}
 		}
 	}
@@ -425,7 +399,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<peer_info::PeerInfoEven
 		for addr in listen_addrs {
 			self.discovery.add_self_reported_address(&peer_id, protocols.iter(), addr);
 		}
-		self.substrate.add_discovered_nodes(iter::once(peer_id));
+		self.substrate.add_default_set_discovered_nodes(iter::once(peer_id));
 	}
 }
 
@@ -440,7 +414,7 @@ impl<B: BlockT, H: ExHashT> NetworkBehaviourEventProcess<DiscoveryOut>
 				// implementation for `PeerInfoEvent`.
 			}
 			DiscoveryOut::Discovered(peer_id) => {
-				self.substrate.add_discovered_nodes(iter::once(peer_id));
+				self.substrate.add_default_set_discovered_nodes(iter::once(peer_id));
 			}
 			DiscoveryOut::ValueFound(results, duration) => {
 				self.events.push_back(BehaviourOut::Dht(DhtEvent::ValueFound(results), duration));

substrate/client/network/src/config.rs

@@ -382,18 +382,12 @@ pub struct NetworkConfiguration {
 	pub boot_nodes: Vec<MultiaddrWithPeerId>,
 	/// The node key configuration, which determines the node's network identity keypair.
 	pub node_key: NodeKeyConfig,
-	/// List of names of notifications protocols that the node supports.
-	pub notifications_protocols: Vec<Cow<'static, str>>,
 	/// List of request-response protocols that the node supports.
 	pub request_response_protocols: Vec<RequestResponseConfig>,
-	/// Maximum allowed number of incoming connections.
+	/// Configuration for the default set of nodes used for block syncing and transactions.
-	pub in_peers: u32,
+	pub default_peers_set: SetConfig,
-	/// Number of outgoing connections we're trying to maintain.
+	/// Configuration for extra sets of nodes.
-	pub out_peers: u32,
+	pub extra_sets: Vec<NonDefaultSetConfig>,
-	/// List of reserved node addresses.
-	pub reserved_nodes: Vec<MultiaddrWithPeerId>,
-	/// The non-reserved peer mode.
-	pub non_reserved_mode: NonReservedPeerMode,
 	/// Client identifier. Sent over the wire for debugging purposes.
 	pub client_version: String,
 	/// Name of the node. Sent over the wire for debugging purposes.
@@ -423,12 +417,9 @@ impl NetworkConfiguration {
 			public_addresses: Vec::new(),
 			boot_nodes: Vec::new(),
 			node_key,
-			notifications_protocols: Vec::new(),
 			request_response_protocols: Vec::new(),
-			in_peers: 25,
+			default_peers_set: Default::default(),
-			out_peers: 75,
+			extra_sets: Vec::new(),
-			reserved_nodes: Vec::new(),
-			non_reserved_mode: NonReservedPeerMode::Accept,
 			client_version: client_version.into(),
 			node_name: node_name.into(),
 			transport: TransportConfig::Normal {
@@ -481,6 +472,44 @@ impl NetworkConfiguration {
 	}
 }
 
+/// Configuration for a set of nodes.
+#[derive(Clone, Debug)]
+pub struct SetConfig {
+	/// Maximum allowed number of incoming substreams related to this set.
+	pub in_peers: u32,
+	/// Number of outgoing substreams related to this set that we're trying to maintain.
+	pub out_peers: u32,
+	/// List of reserved node addresses.
+	pub reserved_nodes: Vec<MultiaddrWithPeerId>,
+	/// Whether nodes that aren't in [`SetConfig::reserved_nodes`] are accepted or automatically
+	/// refused.
+	pub non_reserved_mode: NonReservedPeerMode,
+}
+
+impl Default for SetConfig {
+	fn default() -> Self {
+		SetConfig {
+			in_peers: 25,
+			out_peers: 75,
+			reserved_nodes: Vec::new(),
+			non_reserved_mode: NonReservedPeerMode::Accept,
+		}
+	}
+}
+
+/// Extension to [`SetConfig`] for sets that aren't the default set.
+#[derive(Clone, Debug)]
+pub struct NonDefaultSetConfig {
+	/// Name of the notifications protocols of this set. A substream on this set will be
+	/// considered established once this protocol is open.
+	///
+	/// > **Note**: This field isn't present for the default set, as this is handled internally
+	/// >           by the networking code.
+	pub notifications_protocol: Cow<'static, str>,
+	/// Base configuration.
+	pub set_config: SetConfig,
+}
+
 /// Configuration for the transport layer.
 #[derive(Clone, Debug)]
 pub enum TransportConfig {

substrate/client/network/src/gossip/tests.rs

@@ -141,19 +141,31 @@ fn build_nodes_one_proto()
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![PROTOCOL_NAME],
+		extra_sets: vec![
+			config::NonDefaultSetConfig {
+				notifications_protocol: PROTOCOL_NAME,
+				set_config: Default::default()
+			}
+		],
 		listen_addresses: vec![listen_addr.clone()],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
 
 	let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![PROTOCOL_NAME],
 		listen_addresses: vec![],
-		reserved_nodes: vec![config::MultiaddrWithPeerId {
+		extra_sets: vec![
-			multiaddr: listen_addr,
+			config::NonDefaultSetConfig {
-			peer_id: node1.local_peer_id().clone(),
+				notifications_protocol: PROTOCOL_NAME,
-		}],
+				set_config: config::SetConfig {
+					reserved_nodes: vec![config::MultiaddrWithPeerId {
+						multiaddr: listen_addr,
+						peer_id: node1.local_peer_id().clone(),
+					}],
+					.. Default::default()
+				},
+			}
+		],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});

substrate/client/network/src/light_client_handler.rs

@@ -1301,7 +1301,8 @@ fn fmt_keys(first: Option<&Vec<u8>>, last: Option<&Vec<u8>>) -> String {
 	}
 }
 
-#[cfg(test)]
+// TODO:
+/*#[cfg(test)]
 mod tests {
 	use super::*;
 	use async_std::task;
@@ -2058,4 +2059,4 @@ mod tests {
 				.contains(BlockAttributes::BODY)
 		);
 	}
-}
+}*/

substrate/client/network/src/network_state.rs

@@ -57,12 +57,6 @@ pub struct Peer {
 	pub version_string: Option<String>,
 	/// Latest ping duration with this node.
 	pub latest_ping_time: Option<Duration>,
-	/// If true, the peer is "enabled", which means that we try to open Substrate-related protocols
-	/// with this peer. If false, we stick to Kademlia and/or other network-only protocols.
-	pub enabled: bool,
-	/// If true, the peer is "open", which means that we have a Substrate-related protocol
-	/// with this peer.
-	pub open: bool,
 	/// List of addresses known for this node.
 	pub known_addresses: HashSet<Multiaddr>,
 }

substrate/client/network/src/protocol/event.rs

@@ -48,6 +48,18 @@ pub enum Event {
 	/// Event generated by a DHT.
 	Dht(DhtEvent),
 
+	/// Now connected to a new peer for syncing purposes.
+	SyncConnected {
+		/// Node we are now syncing from.
+		remote: PeerId,
+	},
+
+	/// Now disconnected from a peer for syncing purposes.
+	SyncDisconnected {
+		/// Node we are no longer syncing from.
+		remote: PeerId,
+	},
+
 	/// Opened a substream with the given node with the given notifications protocol.
 	///
 	/// The protocol is always one of the notification protocols that have been registered.

substrate/client/network/src/protocol/generic_proto/tests.rs

@@ -47,7 +47,6 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 
 	for index in 0 .. 2 {
 		let keypair = keypairs[index].clone();
-		let local_peer_id = keypair.public().into_peer_id();
 
 		let noise_keys = noise::Keypair::<noise::X25519Spec>::new()
 			.into_authentic(&keypair)
@@ -61,24 +60,28 @@ fn build_nodes() -> (Swarm<CustomProtoWithAddr>, Swarm<CustomProtoWithAddr>) {
 			.boxed();
 
 		let (peerset, _) = sc_peerset::Peerset::from_config(sc_peerset::PeersetConfig {
-			in_peers: 25,
+			sets: vec![
-			out_peers: 25,
+				sc_peerset::SetConfig {
-			bootnodes: if index == 0 {
+					in_peers: 25,
-				keypairs
+					out_peers: 25,
-					.iter()
+					bootnodes: if index == 0 {
-					.skip(1)
+						keypairs
-					.map(|keypair| keypair.public().into_peer_id())
+							.iter()
-					.collect()
+							.skip(1)
-			} else {
+							.map(|keypair| keypair.public().into_peer_id())
-				vec![]
+							.collect()
-			},
+					} else {
-			reserved_only: false,
+						vec![]
-			priority_groups: Vec::new(),
+					},
+					reserved_nodes: Default::default(),
+					reserved_only: false,
+				}
+			],
 		});
 
 		let behaviour = CustomProtoWithAddr {
 			inner: GenericProto::new(
-				local_peer_id, "test", &[1], vec![], peerset,
+				"test", &[1], vec![], peerset,
 				iter::once(("/foo".into(), Vec::new()))
 			),
 			addrs: addrs
@@ -245,7 +248,10 @@ fn reconnect_after_disconnect() {
 						ServiceState::NotConnected => {
 							service1_state = ServiceState::FirstConnec;
 							if service2_state == ServiceState::FirstConnec {
-								service1.disconnect_peer(Swarm::local_peer_id(&service2));
+								service1.disconnect_peer(
+									Swarm::local_peer_id(&service2),
+									sc_peerset::SetId::from(0)
+								);
 							}
 						},
 						ServiceState::Disconnected => service1_state = ServiceState::ConnectedAgain,
@@ -264,7 +270,10 @@ fn reconnect_after_disconnect() {
 						ServiceState::NotConnected => {
 							service2_state = ServiceState::FirstConnec;
 							if service1_state == ServiceState::FirstConnec {
-								service1.disconnect_peer(Swarm::local_peer_id(&service2));
+								service1.disconnect_peer(
+									Swarm::local_peer_id(&service2),
+									sc_peerset::SetId::from(0)
+								);
 							}
 						},
 						ServiceState::Disconnected => service2_state = ServiceState::ConnectedAgain,

substrate/client/network/src/protocol/generic_proto/upgrade/notifications.rs

@@ -107,11 +107,6 @@ impl NotificationsIn {
 			protocol_name: protocol_name.into(),
 		}
 	}
-
-	/// Returns the name of the protocol that we accept.
-	pub fn protocol_name(&self) -> &Cow<'static, str> {
-		&self.protocol_name
-	}
 }
 
 impl UpgradeInfo for NotificationsIn {

substrate/client/network/src/service.rs

@@ -30,7 +30,7 @@
 use crate::{
 	ExHashT, NetworkStateInfo, NetworkStatus,
 	behaviour::{self, Behaviour, BehaviourOut},
-	config::{parse_str_addr, NonReservedPeerMode, Params, Role, TransportConfig},
+	config::{parse_str_addr, Params, Role, TransportConfig},
 	DhtEvent,
 	discovery::DiscoveryConfig,
 	error::Error,
@@ -147,9 +147,15 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 			&params.network_config.transport,
 		)?;
 		ensure_addresses_consistent_with_transport(
-			params.network_config.reserved_nodes.iter().map(|x| &x.multiaddr),
+			params.network_config.default_peers_set.reserved_nodes.iter().map(|x| &x.multiaddr),
 			&params.network_config.transport,
 		)?;
+		for extra_set in &params.network_config.extra_sets {
+			ensure_addresses_consistent_with_transport(
+				extra_set.set_config.reserved_nodes.iter().map(|x| &x.multiaddr),
+				&params.network_config.transport,
+			)?;
+		}
 		ensure_addresses_consistent_with_transport(
 			params.network_config.public_addresses.iter(),
 			&params.network_config.transport,
@@ -157,12 +163,35 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 
 		let (to_worker, from_service) = tracing_unbounded("mpsc_network_worker");
 
-		if let Some(path) = params.network_config.net_config_path {
+		if let Some(path) = &params.network_config.net_config_path {
-			fs::create_dir_all(&path)?;
+			fs::create_dir_all(path)?;
 		}
 
+		// Private and public keys configuration.
+		let local_identity = params.network_config.node_key.clone().into_keypair()?;
+		let local_public = local_identity.public();
+		let local_peer_id = local_public.clone().into_peer_id();
+		info!(
+			target: "sub-libp2p",
+			"🏷 Local node identity is: {}",
+			local_peer_id.to_base58(),
+		);
+
+		let (protocol, peerset_handle, mut known_addresses) = Protocol::new(
+			protocol::ProtocolConfig {
+				roles: From::from(&params.role),
+				max_parallel_downloads: params.network_config.max_parallel_downloads,
+			},
+			params.chain.clone(),
+			params.transaction_pool,
+			params.protocol_id.clone(),
+			&params.role,
+			&params.network_config,
+			params.block_announce_validator,
+			params.metrics_registry.as_ref(),
+		)?;
+
 		// List of multiaddresses that we know in the network.
-		let mut known_addresses = Vec::new();
 		let mut bootnodes = Vec::new();
 		let mut boot_node_ids = HashSet::new();
 
@@ -192,71 +221,21 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 				}
 			)?;
 
-		// Initialize the peers we should always be connected to.
+		// Print a message about the deprecation of sentry nodes.
-		let priority_groups = {
+		let print_deprecated_message = match &params.role {
-			let mut reserved_nodes = HashSet::new();
+			Role::Sentry { .. } => true,
-			for reserved in params.network_config.reserved_nodes.iter() {
+			Role::Authority { sentry_nodes } if !sentry_nodes.is_empty() => true,
-				reserved_nodes.insert(reserved.peer_id.clone());
+			_ => false,
-				known_addresses.push((reserved.peer_id.clone(), reserved.multiaddr.clone()));
-			}
-
-			let print_deprecated_message = match &params.role {
-				Role::Sentry { .. } => true,
-				Role::Authority { sentry_nodes } if !sentry_nodes.is_empty() => true,
-				_ => false,
-			};
-			if print_deprecated_message {
-				log::warn!(
-					"🙇 Sentry nodes are deprecated, and the `--sentry` and  `--sentry-nodes` \
-					CLI options will eventually be removed in a future version. The Substrate \
-					and Polkadot networking protocol require validators to be \
-					publicly-accessible. Please do not block access to your validator nodes. \
-					For details, see https://github.com/paritytech/substrate/issues/6845."
-				);
-			}
-
-			let mut sentries_and_validators = HashSet::new();
-			match &params.role {
-				Role::Sentry { validators } => {
-					for validator in validators {
-						sentries_and_validators.insert(validator.peer_id.clone());
-						reserved_nodes.insert(validator.peer_id.clone());
-						known_addresses.push((validator.peer_id.clone(), validator.multiaddr.clone()));
-					}
-				}
-				Role::Authority { sentry_nodes } => {
-					for sentry_node in sentry_nodes {
-						sentries_and_validators.insert(sentry_node.peer_id.clone());
-						reserved_nodes.insert(sentry_node.peer_id.clone());
-						known_addresses.push((sentry_node.peer_id.clone(), sentry_node.multiaddr.clone()));
-					}
-				}
-				_ => {}
-			}
-
-			vec![
-				("reserved".to_owned(), reserved_nodes),
-				("sentries_and_validators".to_owned(), sentries_and_validators),
-			]
 		};
-
+		if print_deprecated_message {
-		let peerset_config = sc_peerset::PeersetConfig {
+			log::warn!(
-			in_peers: params.network_config.in_peers,
+				"🙇 Sentry nodes are deprecated, and the `--sentry` and  `--sentry-nodes` \
-			out_peers: params.network_config.out_peers,
+				CLI options will eventually be removed in a future version. The Substrate \
-			bootnodes,
+				and Polkadot networking protocol require validators to be \
-			reserved_only: params.network_config.non_reserved_mode == NonReservedPeerMode::Deny,
+				publicly-accessible. Please do not block access to your validator nodes. \
-			priority_groups,
+				For details, see https://github.com/paritytech/substrate/issues/6845."
-		};
+			);
-
+		}
-		// Private and public keys configuration.
-		let local_identity = params.network_config.node_key.clone().into_keypair()?;
-		let local_public = local_identity.public();
-		let local_peer_id = local_public.clone().into_peer_id();
-		info!(
-			target: "sub-libp2p",
-			"🏷 Local node identity is: {}",
-			local_peer_id.to_base58(),
-		);
 
 		let checker = params.on_demand.as_ref()
 			.map(|od| od.checker().clone())
@@ -264,20 +243,6 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 
 		let num_connected = Arc::new(AtomicUsize::new(0));
 		let is_major_syncing = Arc::new(AtomicBool::new(false));
-		let (protocol, peerset_handle) = Protocol::new(
-			protocol::ProtocolConfig {
-				roles: From::from(&params.role),
-				max_parallel_downloads: params.network_config.max_parallel_downloads,
-			},
-			local_peer_id.clone(),
-			params.chain.clone(),
-			params.transaction_pool,
-			params.protocol_id.clone(),
-			peerset_config,
-			params.block_announce_validator,
-			params.metrics_registry.as_ref(),
-			boot_node_ids.clone(),
-		)?;
 
 		// Build the swarm.
 		let (mut swarm, bandwidth): (Swarm<B, H>, _) = {
@@ -299,7 +264,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 			let discovery_config = {
 				let mut config = DiscoveryConfig::new(local_public.clone());
 				config.with_user_defined(known_addresses);
-				config.discovery_limit(u64::from(params.network_config.out_peers) + 15);
+				config.discovery_limit(u64::from(params.network_config.default_peers_set.out_peers) + 15);
 				config.add_protocol(params.protocol_id.clone());
 				config.allow_non_globals_in_dht(params.network_config.allow_non_globals_in_dht);
 				config.use_kademlia_disjoint_query_paths(params.network_config.kademlia_disjoint_query_paths);
@@ -318,7 +283,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 				config
 			};
 
-			let mut behaviour = {
+			let behaviour = {
 				let result = Behaviour::new(
 					protocol,
 					params.role,
@@ -340,9 +305,6 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 				}
 			};
 
-			for protocol in &params.network_config.notifications_protocols {
-				behaviour.register_notifications_protocol(protocol.clone());
-			}
 			let (transport, bandwidth) = {
 				let (config_mem, config_wasm) = match params.network_config.transport {
 					TransportConfig::MemoryOnly => (true, None),
@@ -551,8 +513,6 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkWorker<B, H> {
 					version_string: swarm.node(peer_id)
 						.and_then(|i| i.client_version().map(|s| s.to_owned())),
 					latest_ping_time: swarm.node(peer_id).and_then(|i| i.latest_ping()),
-					enabled: swarm.user_protocol().is_enabled(&peer_id),
-					open: swarm.user_protocol().is_open(&peer_id),
 					known_addresses,
 				}))
 			}).collect()
@@ -622,7 +582,9 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// Need a better solution to manage authorized peers, but now just use reserved peers for
 	/// prototyping.
 	pub fn set_authorized_peers(&self, peers: HashSet<PeerId>) {
-		self.peerset.set_reserved_peers(peers)
+		let _ = self
+			.to_worker
+			.unbounded_send(ServiceToWorkerMsg::SetReserved(peers));
 	}
 
 	/// Set authorized_only flag.
@@ -630,7 +592,9 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// Need a better solution to decide authorized_only, but now just use reserved_only flag for
 	/// prototyping.
 	pub fn set_authorized_only(&self, reserved_only: bool) {
-		self.peerset.set_reserved_only(reserved_only)
+		let _ = self
+			.to_worker
+			.unbounded_send(ServiceToWorkerMsg::SetReservedOnly(reserved_only));
 	}
 
 	/// Appends a notification to the buffer of pending outgoing notifications with the given peer.
@@ -686,7 +650,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 			message.len()
 		);
 		trace!(target: "sub-libp2p", "Handler({:?}) <= Sync notification", target);
-		sink.send_sync_notification(protocol, message);
+		sink.send_sync_notification(message);
 	}
 
 	/// Obtains a [`NotificationSender`] for a connected peer, if it exists.
@@ -871,8 +835,12 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 	/// Disconnect from a node as soon as possible.
 	///
 	/// This triggers the same effects as if the connection had closed itself spontaneously.
-	pub fn disconnect_peer(&self, who: PeerId) {
+	///
-		let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::DisconnectPeer(who));
+	/// See also [`NetworkService::remove_from_peers_set`], which has the same effect but also
+	/// prevents the local node from re-establishing an outgoing substream to this peer until it
+	/// is added again.
+	pub fn disconnect_peer(&self, who: PeerId, protocol: impl Into<Cow<'static, str>>) {
+		let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::DisconnectPeer(who, protocol.into()));
 	}
 
 	/// Request a justification for the given block from the network.
@@ -910,19 +878,19 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 			.unbounded_send(ServiceToWorkerMsg::PutValue(key, value));
 	}
 
-	/// Connect to unreserved peers and allow unreserved peers to connect.
+	/// Connect to unreserved peers and allow unreserved peers to connect for syncing purposes.
 	pub fn accept_unreserved_peers(&self) {
-		self.peerset.set_reserved_only(false);
+		let _ = self
+			.to_worker
+			.unbounded_send(ServiceToWorkerMsg::SetReservedOnly(false));
 	}
 
-	/// Disconnect from unreserved peers and deny new unreserved peers to connect.
+	/// Disconnect from unreserved peers and deny new unreserved peers to connect for syncing
+	/// purposes.
 	pub fn deny_unreserved_peers(&self) {
-		self.peerset.set_reserved_only(true);
+		let _ = self
-	}
+			.to_worker
-
+			.unbounded_send(ServiceToWorkerMsg::SetReservedOnly(true));
-	/// Removes a `PeerId` from the list of reserved peers.
-	pub fn remove_reserved_peer(&self, peer: PeerId) {
-		self.peerset.remove_reserved_peer(peer);
 	}
 
 	/// Adds a `PeerId` and its address as reserved. The string should encode the address
@@ -936,10 +904,71 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 		if peer_id == self.local_peer_id {
 			return Err("Local peer ID cannot be added as a reserved peer.".to_string())
 		}
-		self.peerset.add_reserved_peer(peer_id.clone());
+
 		let _ = self
 			.to_worker
-			.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr));
+			.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id.clone(), addr));
+		let _ = self
+			.to_worker
+			.unbounded_send(ServiceToWorkerMsg::AddReserved(peer_id));
+		Ok(())
+	}
+
+	/// Removes a `PeerId` from the list of reserved peers.
+	pub fn remove_reserved_peer(&self, peer_id: PeerId) {
+		let _ = self
+			.to_worker
+			.unbounded_send(ServiceToWorkerMsg::RemoveReserved(peer_id));
+	}
+
+	/// Add peers to a peer set.
+	///
+	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. It can also
+	/// consist of only `/p2p/<peerid>`.
+	///
+	/// Returns an `Err` if one of the given addresses is invalid or contains an
+	/// invalid peer ID (which includes the local peer ID).
+	pub fn add_peers_to_reserved_set(&self, protocol: Cow<'static, str>, peers: HashSet<Multiaddr>) -> Result<(), String> {
+		let peers = self.split_multiaddr_and_peer_id(peers)?;
+
+		for (peer_id, addr) in peers.into_iter() {
+			// Make sure the local peer ID is never added to the PSM.
+			if peer_id == self.local_peer_id {
+				return Err("Local peer ID cannot be added as a reserved peer.".to_string())
+			}
+
+			if !addr.is_empty() {
+				let _ = self
+					.to_worker
+					.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id.clone(), addr));
+			}
+			let _ = self
+				.to_worker
+				.unbounded_send(ServiceToWorkerMsg::AddSetReserved(protocol.clone(), peer_id));
+		}
+
+		Ok(())
+	}
+
+	/// Remove peers from a peer set.
+	///
+	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`.
+	///
+	/// Returns an `Err` if one of the given addresses is invalid or contains an
+	/// invalid peer ID (which includes the local peer ID).
+	//
+	// NOTE: technically, this function only needs `Vec<PeerId>`, but we use `Multiaddr` here for convenience.
+	pub fn remove_peers_from_reserved_set(
+		&self,
+		protocol: Cow<'static, str>,
+		peers: HashSet<Multiaddr>
+	) -> Result<(), String> {
+		let peers = self.split_multiaddr_and_peer_id(peers)?;
+		for (peer_id, _) in peers.into_iter() {
+			let _ = self
+				.to_worker
+				.unbounded_send(ServiceToWorkerMsg::RemoveSetReserved(protocol.clone(), peer_id));
+		}
 		Ok(())
 	}
 
@@ -955,68 +984,53 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 			.unbounded_send(ServiceToWorkerMsg::SyncFork(peers, hash, number));
 	}
 
-	/// Modify a peerset priority group.
+	/// Add a peer to a set of peers.
 	///
-	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`.
+	/// If the set has slots available, it will try to open a substream with this peer.
+	///
+	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. It can also
+	/// consist of only `/p2p/<peerid>`.
 	///
 	/// Returns an `Err` if one of the given addresses is invalid or contains an
 	/// invalid peer ID (which includes the local peer ID).
-	//
+	pub fn add_to_peers_set(&self, protocol: Cow<'static, str>, peers: HashSet<Multiaddr>) -> Result<(), String> {
-	// NOTE: even though this function is currently sync, it's marked as async for
-	// future-proofing, see https://github.com/paritytech/substrate/pull/7247#discussion_r502263451.
-	pub async fn set_priority_group(&self, group_id: String, peers: HashSet<Multiaddr>) -> Result<(), String> {
 		let peers = self.split_multiaddr_and_peer_id(peers)?;
 
-		let peer_ids = peers.iter().map(|(peer_id, _addr)| peer_id.clone()).collect();
-
-		self.peerset.set_priority_group(group_id, peer_ids);
-
 		for (peer_id, addr) in peers.into_iter() {
+			// Make sure the local peer ID is never added to the PSM.
+			if peer_id == self.local_peer_id {
+				return Err("Local peer ID cannot be added as a reserved peer.".to_string())
+			}
+
+			if !addr.is_empty() {
+				let _ = self
+					.to_worker
+					.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id.clone(), addr));
+			}
 			let _ = self
 				.to_worker
-				.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr));
+				.unbounded_send(ServiceToWorkerMsg::AddToPeersSet(protocol.clone(), peer_id));
 		}
 
 		Ok(())
 	}
 
-	/// Add peers to a peerset priority group.
+	/// Remove peers from a peer set.
+	///
+	/// If we currently have an open substream with this peer, it will soon be closed.
 	///
 	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`.
 	///
 	/// Returns an `Err` if one of the given addresses is invalid or contains an
 	/// invalid peer ID (which includes the local peer ID).
 	//
-	// NOTE: even though this function is currently sync, it's marked as async for
-	// future-proofing, see https://github.com/paritytech/substrate/pull/7247#discussion_r502263451.
-	pub async fn add_to_priority_group(&self, group_id: String, peers: HashSet<Multiaddr>) -> Result<(), String> {
-		let peers = self.split_multiaddr_and_peer_id(peers)?;
-
-		for (peer_id, addr) in peers.into_iter() {
-			self.peerset.add_to_priority_group(group_id.clone(), peer_id.clone());
-
-			let _ = self
-				.to_worker
-				.unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr));
-		}
-
-		Ok(())
-	}
-
-	/// Remove peers from a peerset priority group.
-	///
-	/// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`.
-	///
-	/// Returns an `Err` if one of the given addresses is invalid or contains an
-	/// invalid peer ID (which includes the local peer ID).
-	//
-	// NOTE: even though this function is currently sync, it's marked as async for
-	// future-proofing, see https://github.com/paritytech/substrate/pull/7247#discussion_r502263451.
 	// NOTE: technically, this function only needs `Vec<PeerId>`, but we use `Multiaddr` here for convenience.
-	pub async fn remove_from_priority_group(&self, group_id: String, peers: HashSet<Multiaddr>) -> Result<(), String> {
+	pub fn remove_from_peers_set(&self, protocol: Cow<'static, str>, peers: HashSet<Multiaddr>) -> Result<(), String> {
 		let peers = self.split_multiaddr_and_peer_id(peers)?;
 		for (peer_id, _) in peers.into_iter() {
-			self.peerset.remove_from_priority_group(group_id.clone(), peer_id);
+			let _ = self
+				.to_worker
+				.unbounded_send(ServiceToWorkerMsg::RemoveFromPeersSet(protocol.clone(), peer_id));
 		}
 		Ok(())
 	}
@@ -1033,7 +1047,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 			.unbounded_send(ServiceToWorkerMsg::NewBestBlockImported(hash, number));
 	}
 
-	/// Utility function to extract `PeerId` from each `Multiaddr` for priority group updates.
+	/// Utility function to extract `PeerId` from each `Multiaddr` for peer set updates.
 	///
 	/// Returns an `Err` if one of the given addresses is invalid or contains an
 	/// invalid peer ID (which includes the local peer ID).
@@ -1049,7 +1063,7 @@ impl<B: BlockT + 'static, H: ExHashT> NetworkService<B, H> {
 				// Make sure the local peer ID is never added to the PSM
 				// or added as a "known address", even if given.
 				if peer == self.local_peer_id {
-					Err("Local peer ID in priority group.".to_string())
+					Err("Local peer ID in peer set.".to_string())
 				} else {
 					Ok((peer, addr))
 				}
@@ -1115,11 +1129,12 @@ impl NotificationSender {
 	/// Returns a future that resolves when the `NotificationSender` is ready to send a notification.
 	pub async fn ready<'a>(&'a self) -> Result<NotificationSenderReady<'a>, NotificationSenderError> {
 		Ok(NotificationSenderReady {
-			ready: match self.sink.reserve_notification(self.protocol_name.clone()).await {
+			ready: match self.sink.reserve_notification().await {
 				Ok(r) => r,
 				Err(()) => return Err(NotificationSenderError::Closed),
 			},
 			peer_id: self.sink.peer_id(),
+			protocol_name: &self.protocol_name,
 			notification_size_metric: self.notification_size_metric.clone(),
 		})
 	}
@@ -1133,6 +1148,9 @@ pub struct NotificationSenderReady<'a> {
 	/// Target of the notification.
 	peer_id: &'a PeerId,
 
+	/// Name of the protocol on the wire.
+	protocol_name: &'a Cow<'static, str>,
+
 	/// Field extracted from the [`Metrics`] struct and necessary to report the
 	/// notifications-related metrics.
 	notification_size_metric: Option<Histogram>,
@@ -1149,9 +1167,9 @@ impl<'a> NotificationSenderReady<'a> {
 
 		trace!(
 			target: "sub-libp2p",
-			"External API => Notification({:?}, {:?}, {} bytes)",
+			"External API => Notification({:?}, {}, {} bytes)",
 			self.peer_id,
-			self.ready.protocol_name(),
+			self.protocol_name,
 			notification.len()
 		);
 		trace!(target: "sub-libp2p", "Handler({:?}) <= Async notification", self.peer_id);
@@ -1186,6 +1204,14 @@ enum ServiceToWorkerMsg<B: BlockT, H: ExHashT> {
 	GetValue(record::Key),
 	PutValue(record::Key, Vec<u8>),
 	AddKnownAddress(PeerId, Multiaddr),
+	SetReservedOnly(bool),
+	AddReserved(PeerId),
+	RemoveReserved(PeerId),
+	SetReserved(HashSet<PeerId>),
+	AddSetReserved(Cow<'static, str>, PeerId),
+	RemoveSetReserved(Cow<'static, str>, PeerId),
+	AddToPeersSet(Cow<'static, str>, PeerId),
+	RemoveFromPeersSet(Cow<'static, str>, PeerId),
 	SyncFork(Vec<PeerId>, B::Hash, NumberFor<B>),
 	EventStream(out_events::Sender),
 	Request {
@@ -1194,7 +1220,7 @@ enum ServiceToWorkerMsg<B: BlockT, H: ExHashT> {
 		request: Vec<u8>,
 		pending_response: oneshot::Sender<Result<Vec<u8>, RequestFailure>>,
 	},
-	DisconnectPeer(PeerId),
+	DisconnectPeer(PeerId, Cow<'static, str>),
 	NewBestBlockImported(B::Hash, NumberFor<B>),
 }
 
@@ -1290,8 +1316,24 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 					this.network_service.get_value(&key),
 				ServiceToWorkerMsg::PutValue(key, value) =>
 					this.network_service.put_value(key, value),
+				ServiceToWorkerMsg::SetReservedOnly(reserved_only) =>
+					this.network_service.user_protocol_mut().set_reserved_only(reserved_only),
+				ServiceToWorkerMsg::SetReserved(peers) =>
+					this.network_service.user_protocol_mut().set_reserved_peers(peers),
+				ServiceToWorkerMsg::AddReserved(peer_id) =>
+					this.network_service.user_protocol_mut().add_reserved_peer(peer_id),
+				ServiceToWorkerMsg::RemoveReserved(peer_id) =>
+					this.network_service.user_protocol_mut().remove_reserved_peer(peer_id),
+				ServiceToWorkerMsg::AddSetReserved(protocol, peer_id) =>
+					this.network_service.user_protocol_mut().add_set_reserved_peer(protocol, peer_id),
+				ServiceToWorkerMsg::RemoveSetReserved(protocol, peer_id) =>
+					this.network_service.user_protocol_mut().remove_set_reserved_peer(protocol, peer_id),
 				ServiceToWorkerMsg::AddKnownAddress(peer_id, addr) =>
 					this.network_service.add_known_address(peer_id, addr),
+				ServiceToWorkerMsg::AddToPeersSet(protocol, peer_id) =>
+					this.network_service.user_protocol_mut().add_to_peers_set(protocol, peer_id),
+				ServiceToWorkerMsg::RemoveFromPeersSet(protocol, peer_id) =>
+					this.network_service.user_protocol_mut().remove_from_peers_set(protocol, peer_id),
 				ServiceToWorkerMsg::SyncFork(peer_ids, hash, number) =>
 					this.network_service.user_protocol_mut().set_sync_fork_request(peer_ids, &hash, number),
 				ServiceToWorkerMsg::EventStream(sender) =>
@@ -1299,8 +1341,8 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 				ServiceToWorkerMsg::Request { target, protocol, request, pending_response } => {
 					this.network_service.send_request(&target, &protocol, request, pending_response);
 				},
-				ServiceToWorkerMsg::DisconnectPeer(who) =>
+				ServiceToWorkerMsg::DisconnectPeer(who, protocol_name) =>
-					this.network_service.user_protocol_mut().disconnect_peer(&who),
+					this.network_service.user_protocol_mut().disconnect_peer(&who, &protocol_name),
 				ServiceToWorkerMsg::NewBestBlockImported(hash, number) =>
 					this.network_service.user_protocol_mut().new_best_block_imported(hash, number),
 			}
@@ -1479,6 +1521,12 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 						messages,
 					});
 				},
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::SyncConnected(remote))) => {
+					this.event_streams.send(Event::SyncConnected { remote });
+				},
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::SyncDisconnected(remote))) => {
+					this.event_streams.send(Event::SyncDisconnected { remote });
+				},
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::Dht(event, duration))) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						let query_type = match event {
@@ -1702,12 +1750,7 @@ impl<'a, B: BlockT, H: ExHashT> Link<B> for NetworkLink<'a, B, H> {
 		self.protocol.user_protocol_mut().on_blocks_processed(imported, count, results)
 	}
 	fn justification_imported(&mut self, who: PeerId, hash: &B::Hash, number: NumberFor<B>, success: bool) {
-		self.protocol.user_protocol_mut().justification_import_result(hash.clone(), number, success);
+		self.protocol.user_protocol_mut().justification_import_result(who, hash.clone(), number, success);
-		if !success {
-			info!("💔 Invalid justification provided by {} for #{}", who, hash);
-			self.protocol.user_protocol_mut().disconnect_peer(&who);
-			self.protocol.user_protocol_mut().report_peer(who, ReputationChange::new_fatal("Invalid justification"));
-		}
 	}
 	fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>) {
 		self.protocol.user_protocol_mut().request_justification(hash, number)

substrate/client/network/src/service/out_events.rs

@@ -227,6 +227,16 @@ impl Metrics {
 					.with_label_values(&["dht", "sent", name])
 					.inc_by(num);
 			}
+			Event::SyncConnected { .. } => {
+				self.events_total
+					.with_label_values(&["sync-connected", "sent", name])
+					.inc_by(num);
+			}
+			Event::SyncDisconnected { .. } => {
+				self.events_total
+					.with_label_values(&["sync-disconnected", "sent", name])
+					.inc_by(num);
+			}
 			Event::NotificationStreamOpened { protocol, .. } => {
 				self.events_total
 					.with_label_values(&[&format!("notif-open-{:?}", protocol), "sent", name])
@@ -257,6 +267,16 @@ impl Metrics {
 					.with_label_values(&["dht", "received", name])
 					.inc();
 			}
+			Event::SyncConnected { .. } => {
+				self.events_total
+					.with_label_values(&["sync-connected", "received", name])
+					.inc();
+			}
+			Event::SyncDisconnected { .. } => {
+				self.events_total
+					.with_label_values(&["sync-disconnected", "received", name])
+					.inc();
+			}
 			Event::NotificationStreamOpened { protocol, .. } => {
 				self.events_total
 					.with_label_values(&[&format!("notif-open-{:?}", protocol), "received", name])

substrate/client/network/src/service/tests.rs

@@ -141,19 +141,31 @@ fn build_nodes_one_proto()
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![PROTOCOL_NAME],
+		extra_sets: vec![
+			config::NonDefaultSetConfig {
+				notifications_protocol: PROTOCOL_NAME,
+				set_config: Default::default()
+			}
+		],
 		listen_addresses: vec![listen_addr.clone()],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
 
 	let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![PROTOCOL_NAME],
+		extra_sets: vec![
+			config::NonDefaultSetConfig {
+				notifications_protocol: PROTOCOL_NAME,
+				set_config: config::SetConfig {
+					reserved_nodes: vec![config::MultiaddrWithPeerId {
+						multiaddr: listen_addr,
+						peer_id: node1.local_peer_id().clone(),
+					}],
+					.. Default::default()
+				}
+			}
+		],
 		listen_addresses: vec![],
-		reserved_nodes: vec![config::MultiaddrWithPeerId {
-			multiaddr: listen_addr,
-			peer_id: node1.local_peer_id().clone(),
-		}],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
@@ -205,10 +217,10 @@ fn notifications_state_consistent() {
 
 			// Also randomly disconnect the two nodes from time to time.
 			if rand::random::<u8>() % 20 == 0 {
-				node1.disconnect_peer(node2.local_peer_id().clone());
+				node1.disconnect_peer(node2.local_peer_id().clone(), PROTOCOL_NAME);
 			}
 			if rand::random::<u8>() % 20 == 0 {
-				node2.disconnect_peer(node1.local_peer_id().clone());
+				node2.disconnect_peer(node1.local_peer_id().clone(), PROTOCOL_NAME);
 			}
 
 			// Grab next event from either `events_stream1` or `events_stream2`.
@@ -279,6 +291,10 @@ fn notifications_state_consistent() {
 				}
 
 				// Add new events here.
+				future::Either::Left(Event::SyncConnected { .. }) => {}
+				future::Either::Right(Event::SyncConnected { .. }) => {}
+				future::Either::Left(Event::SyncDisconnected { .. }) => {}
+				future::Either::Right(Event::SyncDisconnected { .. }) => {}
 				future::Either::Left(Event::Dht(_)) => {}
 				future::Either::Right(Event::Dht(_)) => {}
 			};
@@ -291,9 +307,16 @@ fn lots_of_incoming_peers_works() {
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 
 	let (main_node, _) = build_test_full_node(config::NetworkConfiguration {
-		notifications_protocols: vec![PROTOCOL_NAME],
 		listen_addresses: vec![listen_addr.clone()],
-		in_peers: u32::max_value(),
+		extra_sets: vec![
+			config::NonDefaultSetConfig {
+				notifications_protocol: PROTOCOL_NAME,
+				set_config: config::SetConfig {
+					in_peers: u32::max_value(),
+					.. Default::default()
+				},
+			}
+		],
 		transport: config::TransportConfig::MemoryOnly,
 		.. config::NetworkConfiguration::new_local()
 	});
@@ -308,12 +331,19 @@ fn lots_of_incoming_peers_works() {
 		let main_node_peer_id = main_node_peer_id.clone();
 
 		let (_dialing_node, event_stream) = build_test_full_node(config::NetworkConfiguration {
-			notifications_protocols: vec![PROTOCOL_NAME],
 			listen_addresses: vec![],
-			reserved_nodes: vec![config::MultiaddrWithPeerId {
+			extra_sets: vec![
-				multiaddr: listen_addr.clone(),
+				config::NonDefaultSetConfig {
-				peer_id: main_node_peer_id.clone(),
+					notifications_protocol: PROTOCOL_NAME,
-			}],
+					set_config: config::SetConfig {
+						reserved_nodes: vec![config::MultiaddrWithPeerId {
+							multiaddr: listen_addr.clone(),
+							peer_id: main_node_peer_id.clone(),
+						}],
+						.. Default::default()
+					},
+				}
+			],
 			transport: config::TransportConfig::MemoryOnly,
 			.. config::NetworkConfiguration::new_local()
 		});
@@ -475,7 +505,10 @@ fn ensure_reserved_node_addresses_consistent_with_transport_memory() {
 	let _ = build_test_full_node(config::NetworkConfiguration {
 		listen_addresses: vec![listen_addr.clone()],
 		transport: config::TransportConfig::MemoryOnly,
-		reserved_nodes: vec![reserved_node],
+		default_peers_set: config::SetConfig {
+			reserved_nodes: vec![reserved_node],
+			.. Default::default()
+		},
 		.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
 	});
 }
@@ -491,7 +524,10 @@ fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() {
 
 	let _ = build_test_full_node(config::NetworkConfiguration {
 		listen_addresses: vec![listen_addr.clone()],
-		reserved_nodes: vec![reserved_node],
+		default_peers_set: config::SetConfig {
+			reserved_nodes: vec![reserved_node],
+			.. Default::default()
+		},
 		.. config::NetworkConfiguration::new("test-node", "test-client", Default::default(), None)
 	});
 }

substrate/client/network/test/src/lib.rs

@@ -52,7 +52,7 @@ use sp_consensus::{BlockOrigin, ForkChoiceStrategy, BlockImportParams, BlockChec
 use futures::prelude::*;
 use futures::future::BoxFuture;
 use sc_network::{NetworkWorker, NetworkService, config::ProtocolId};
-use sc_network::config::{NetworkConfiguration, TransportConfig};
+use sc_network::config::{NetworkConfiguration, NonDefaultSetConfig, TransportConfig};
 use libp2p::PeerId;
 use parking_lot::Mutex;
 use sp_core::H256;
@@ -682,7 +682,12 @@ pub trait TestNetFactory: Sized {
 		network_config.transport = TransportConfig::MemoryOnly;
 		network_config.listen_addresses = vec![listen_addr.clone()];
 		network_config.allow_non_globals_in_dht = true;
-		network_config.notifications_protocols = config.notifications_protocols;
+		network_config.extra_sets = config.notifications_protocols.into_iter().map(|p| {
+			NonDefaultSetConfig {
+				notifications_protocol: p,
+				set_config: Default::default()
+			}
+		}).collect();
 
 		let protocol_id = ProtocolId::from("test-protocol-name");
 

substrate/client/peerset/src/lib.rs

@@ -18,14 +18,27 @@
 
 //! Peer Set Manager (PSM). Contains the strategy for choosing which nodes the network should be
 //! connected to.
+//!
+//! The PSM handles *sets* of nodes. A set of nodes is defined as the nodes that are believed to
+//! support a certain capability, such as handling blocks and transactions of a specific chain,
+//! or collating a certain parachain.
+//!
+//! For each node in each set, the peerset holds a flag specifying whether the node is
+//! connected to us or not.
+//!
+//! This connected/disconnected status is specific to the node and set combination, and it is for
+//! example possible for a node to be connected through a specific set but not another.
+//!
+//! In addition, for each, set, the peerset also holds a list of reserved nodes towards which it
+//! will at all time try to maintain a connection with.
 
 mod peersstate;
 
-use std::{collections::{HashSet, HashMap}, collections::VecDeque};
+use std::{collections::HashSet, collections::VecDeque};
 use futures::prelude::*;
 use log::{debug, error, trace};
 use serde_json::json;
-use std::{pin::Pin, task::{Context, Poll}, time::Duration};
+use std::{collections::HashMap, pin::Pin, task::{Context, Poll}, time::Duration};
 use wasm_timer::Instant;
 use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver};
 
@@ -35,22 +48,46 @@ pub use libp2p::PeerId;
 const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100);
 /// Reputation change for a node when we get disconnected from it.
 const DISCONNECT_REPUTATION_CHANGE: i32 = -256;
-/// Reserved peers group ID
-const RESERVED_NODES: &str = "reserved";
 /// Amount of time between the moment we disconnect from a node and the moment we remove it from
 /// the list.
 const FORGET_AFTER: Duration = Duration::from_secs(3600);
 
 #[derive(Debug)]
 enum Action {
-	AddReservedPeer(PeerId),
+	AddReservedPeer(SetId, PeerId),
-	RemoveReservedPeer(PeerId),
+	RemoveReservedPeer(SetId, PeerId),
-	SetReservedPeers(HashSet<PeerId>),
+	SetReservedPeers(SetId, HashSet<PeerId>),
-	SetReservedOnly(bool),
+	SetReservedOnly(SetId, bool),
 	ReportPeer(PeerId, ReputationChange),
-	SetPriorityGroup(String, HashSet<PeerId>),
+	AddToPeersSet(SetId, PeerId),
-	AddToPriorityGroup(String, PeerId),
+	RemoveFromPeersSet(SetId, PeerId),
-	RemoveFromPriorityGroup(String, PeerId),
+}
+
+/// Identifier of a set in the peerset.
+///
+/// Can be constructed using the `From<usize>` trait implementation based on the index of the set
+/// within [`PeersetConfig::sets`]. For example, the first element of [`PeersetConfig::sets`] is
+/// later referred to with `SetId::from(0)`. It is intended that the code responsible for building
+/// the [`PeersetConfig`] is also responsible for constructing the [`SetId`]s.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct SetId(usize);
+
+impl SetId {
+	pub const fn from(id: usize) -> Self {
+		SetId(id)
+	}
+}
+
+impl From<usize> for SetId {
+	fn from(id: usize) -> Self {
+		SetId(id)
+	}
+}
+
+impl From<SetId> for usize {
+	fn from(id: SetId) -> Self {
+		id.0
+	}
 }
 
 /// Description of a reputation adjustment for a node.
@@ -88,25 +125,26 @@ impl PeersetHandle {
 	///
 	/// > **Note**: Keep in mind that the networking has to know an address for this node,
 	/// >           otherwise it will not be able to connect to it.
-	pub fn add_reserved_peer(&self, peer_id: PeerId) {
+	pub fn add_reserved_peer(&self, set_id: SetId, peer_id: PeerId) {
-		let _ = self.tx.unbounded_send(Action::AddReservedPeer(peer_id));
+		let _ = self.tx.unbounded_send(Action::AddReservedPeer(set_id, peer_id));
 	}
 
 	/// Remove a previously-added reserved peer.
 	///
 	/// Has no effect if the node was not a reserved peer.
-	pub fn remove_reserved_peer(&self, peer_id: PeerId) {
+	pub fn remove_reserved_peer(&self, set_id: SetId, peer_id: PeerId) {
-		let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(peer_id));
+		let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(set_id, peer_id));
 	}
 
-	/// Sets whether or not the peerset only has connections .
+	/// Sets whether or not the peerset only has connections with nodes marked as reserved for
-	pub fn set_reserved_only(&self, reserved: bool) {
+	/// the given set.
-		let _ = self.tx.unbounded_send(Action::SetReservedOnly(reserved));
+	pub fn set_reserved_only(&self, set_id: SetId, reserved: bool) {
+		let _ = self.tx.unbounded_send(Action::SetReservedOnly(set_id, reserved));
 	}
 
 	/// Set reserved peers to the new set.
-	pub fn set_reserved_peers(&self, peer_ids: HashSet<PeerId>) {
+	pub fn set_reserved_peers(&self, set_id: SetId, peer_ids: HashSet<PeerId>) {
-		let _ = self.tx.unbounded_send(Action::SetReservedPeers(peer_ids));
+		let _ = self.tx.unbounded_send(Action::SetReservedPeers(set_id, peer_ids));
 	}
 
 	/// Reports an adjustment to the reputation of the given peer.
@@ -114,19 +152,14 @@ impl PeersetHandle {
 		let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff));
 	}
 
-	/// Modify a priority group.
+	/// Add a peer to a set.
-	pub fn set_priority_group(&self, group_id: String, peers: HashSet<PeerId>) {
+	pub fn add_to_peers_set(&self, set_id: SetId, peer_id: PeerId) {
-		let _ = self.tx.unbounded_send(Action::SetPriorityGroup(group_id, peers));
+		let _ = self.tx.unbounded_send(Action::AddToPeersSet(set_id, peer_id));
 	}
 
-	/// Add a peer to a priority group.
+	/// Remove a peer from a set.
-	pub fn add_to_priority_group(&self, group_id: String, peer_id: PeerId) {
+	pub fn remove_from_peers_set(&self, set_id: SetId, peer_id: PeerId) {
-		let _ = self.tx.unbounded_send(Action::AddToPriorityGroup(group_id, peer_id));
+		let _ = self.tx.unbounded_send(Action::RemoveFromPeersSet(set_id, peer_id));
-	}
-
-	/// Remove a peer from a priority group.
-	pub fn remove_from_priority_group(&self, group_id: String, peer_id: PeerId) {
-		let _ = self.tx.unbounded_send(Action::RemoveFromPriorityGroup(group_id, peer_id));
 	}
 }
 
@@ -135,10 +168,18 @@ impl PeersetHandle {
 pub enum Message {
 	/// Request to open a connection to the given peer. From the point of view of the PSM, we are
 	/// immediately connected.
-	Connect(PeerId),
+	Connect {
+		set_id: SetId,
+		/// Peer to connect to.
+		peer_id: PeerId,
+	},
 
 	/// Drop the connection to the given peer, or cancel the connection attempt after a `Connect`.
-	Drop(PeerId),
+	Drop {
+		set_id: SetId,
+		/// Peer to disconnect from.
+		peer_id: PeerId,
+	},
 
 	/// Equivalent to `Connect` for the peer corresponding to this incoming index.
 	Accept(IncomingIndex),
@@ -160,26 +201,33 @@ impl From<u64> for IncomingIndex {
 /// Configuration to pass when creating the peer set manager.
 #[derive(Debug)]
 pub struct PeersetConfig {
+	/// List of sets of nodes the peerset manages.
+	pub sets: Vec<SetConfig>,
+}
+
+/// Configuration for a single set of nodes.
+#[derive(Debug)]
+pub struct SetConfig {
 	/// Maximum number of ingoing links to peers.
 	pub in_peers: u32,
 
 	/// Maximum number of outgoing links to peers.
 	pub out_peers: u32,
 
-	/// List of bootstrap nodes to initialize the peer with.
+	/// List of bootstrap nodes to initialize the set with.
 	///
 	/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
 	/// >           otherwise it will not be able to connect to them.
 	pub bootnodes: Vec<PeerId>,
 
-	/// If true, we only accept nodes in [`PeersetConfig::priority_groups`].
-	pub reserved_only: bool,
-
 	/// Lists of nodes we should always be connected to.
 	///
 	/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
-	/// >           otherwise it will not be able to connect to them.
+	/// >			otherwise it will not be able to connect to them.
-	pub priority_groups: Vec<(String, HashSet<PeerId>)>,
+	pub reserved_nodes: HashSet<PeerId>,
+
+	/// If true, we only accept nodes in [`SetConfig::reserved_nodes`].
+	pub reserved_only: bool,
 }
 
 /// Side of the peer set manager owned by the network. In other words, the "receiving" side.
@@ -190,11 +238,10 @@ pub struct PeersetConfig {
 pub struct Peerset {
 	/// Underlying data structure for the nodes's states.
 	data: peersstate::PeersState,
-	/// If true, we only accept reserved nodes.
+	/// For each set, lists of nodes that don't occupy slots and that we should try to always be
-	reserved_only: bool,
+	/// connected to, and whether only reserved nodes are accepted. Is kept in sync with the list
-	/// Lists of nodes that don't occupy slots and that we should try to always be connected to.
+	/// of non-slot-occupying nodes in [`Peerset::data`].
-	/// Is kept in sync with the list of reserved nodes in [`Peerset::data`].
+	reserved_nodes: Vec<(HashSet<PeerId>, bool)>,
-	priority_groups: HashMap<String, HashSet<PeerId>>,
 	/// Receiver for messages from the `PeersetHandle` and from `tx`.
 	rx: TracingUnboundedReceiver<Action>,
 	/// Sending side of `rx`.
@@ -216,28 +263,36 @@ impl Peerset {
 			tx: tx.clone(),
 		};
 
-		let now = Instant::now();
+		let mut peerset = {
+			let now = Instant::now();
 
-		let mut peerset = Peerset {
+			Peerset {
-			data: peersstate::PeersState::new(config.in_peers, config.out_peers),
+				data: peersstate::PeersState::new(config.sets.iter().map(|set| peersstate::SetConfig {
-			tx,
+					in_peers: set.in_peers,
-			rx,
+					out_peers: set.out_peers,
-			reserved_only: config.reserved_only,
+				})),
-			priority_groups: config.priority_groups.clone().into_iter().collect(),
+				tx,
-			message_queue: VecDeque::new(),
+				rx,
-			created: now,
+				reserved_nodes: config.sets.iter().map(|set| {
-			latest_time_update: now,
+					(set.reserved_nodes.clone(), set.reserved_only)
+				}).collect(),
+				message_queue: VecDeque::new(),
+				created: now,
+				latest_time_update: now,
+			}
 		};
 
-		for node in config.priority_groups.into_iter().flat_map(|(_, l)| l) {
+		for (set, set_config) in config.sets.into_iter().enumerate() {
-			peerset.data.add_no_slot_node(node);
+			for node in set_config.reserved_nodes {
-		}
+				peerset.data.add_no_slot_node(set, node);
+			}
 
-		for peer_id in config.bootnodes {
+			for peer_id in set_config.bootnodes {
-			if let peersstate::Peer::Unknown(entry) = peerset.data.peer(&peer_id) {
+				if let peersstate::Peer::Unknown(entry) = peerset.data.peer(set, &peer_id) {
-				entry.discover();
+					entry.discover();
-			} else {
+				} else {
-				debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id);
+					debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id);
+				}
 			}
 		}
 
@@ -245,96 +300,109 @@ impl Peerset {
 		(peerset, handle)
 	}
 
-	fn on_add_reserved_peer(&mut self, peer_id: PeerId) {
+	fn on_add_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) {
-		self.on_add_to_priority_group(RESERVED_NODES, peer_id);
+		let newly_inserted = self.reserved_nodes[set_id.0].0.insert(peer_id.clone());
+		if !newly_inserted {
+			return;
+		}
+
+		self.data.add_no_slot_node(set_id.0, peer_id);
+		self.alloc_slots();
 	}
 
-	fn on_remove_reserved_peer(&mut self, peer_id: PeerId) {
+	fn on_remove_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) {
-		self.on_remove_from_priority_group(RESERVED_NODES, peer_id);
+		if !self.reserved_nodes[set_id.0].0.remove(&peer_id) {
-	}
+			return;
+		}
 
-	fn on_set_reserved_peers(&mut self, peer_ids: HashSet<PeerId>) {
+		self.data.remove_no_slot_node(set_id.0, &peer_id);
-		self.on_set_priority_group(RESERVED_NODES, peer_ids);
-	}
 
-	fn on_set_reserved_only(&mut self, reserved_only: bool) {
+		// Nothing more to do if not in reserved-only mode.
-		self.reserved_only = reserved_only;
+		if !self.reserved_nodes[set_id.0].1 {
+			return;
+		}
 
-		if self.reserved_only {
+		// If, however, the peerset is in reserved-only mode, then the removed node needs to be
-			// Disconnect all the nodes that aren't reserved.
+		// disconnected.
-			for peer_id in self.data.connected_peers().cloned().collect::<Vec<_>>().into_iter() {
+		if let peersstate::Peer::Connected(peer) = self.data.peer(set_id.0, &peer_id) {
-				if self.priority_groups.get(RESERVED_NODES).map_or(false, |g| g.contains(&peer_id)) {
+			peer.disconnect();
-					continue;
+			self.message_queue.push_back(Message::Drop {
-				}
+				set_id,
-
+				peer_id,
-				let peer = self.data.peer(&peer_id).into_connected()
+			});
-					.expect("We are enumerating connected peers, therefore the peer is connected; qed");
-				peer.disconnect();
-				self.message_queue.push_back(Message::Drop(peer_id));
-			}
-
-		} else {
-			self.alloc_slots();
 		}
 	}
 
-	fn on_set_priority_group(&mut self, group_id: &str, peers: HashSet<PeerId>) {
+	fn on_set_reserved_peers(&mut self, set_id: SetId, peer_ids: HashSet<PeerId>) {
 		// Determine the difference between the current group and the new list.
 		let (to_insert, to_remove) = {
-			let current_group = self.priority_groups.entry(group_id.to_owned()).or_default();
+			let to_insert = peer_ids.difference(&self.reserved_nodes[set_id.0].0)
-			let to_insert = peers.difference(current_group)
 				.cloned().collect::<Vec<_>>();
-			let to_remove = current_group.difference(&peers)
+			let to_remove = self.reserved_nodes[set_id.0].0.difference(&peer_ids)
 				.cloned().collect::<Vec<_>>();
 			(to_insert, to_remove)
 		};
 
-		// Enumerate elements in `peers` not in `current_group`.
+		for node in to_insert {
-		for peer_id in &to_insert {
+			self.on_add_reserved_peer(set_id, node);
-			// We don't call `on_add_to_priority_group` here in order to avoid calling
-			// `alloc_slots` all the time.
-			self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone());
-			self.data.add_no_slot_node(peer_id.clone());
 		}
 
-		// Enumerate elements in `current_group` not in `peers`.
+		for node in to_remove {
-		for peer in to_remove {
+			self.on_remove_reserved_peer(set_id, node);
-			self.on_remove_from_priority_group(group_id, peer);
 		}
+	}
 
-		if !to_insert.is_empty() {
+	fn on_set_reserved_only(&mut self, set_id: SetId, reserved_only: bool) {
+		self.reserved_nodes[set_id.0].1 = reserved_only;
+
+		if reserved_only {
+			// Disconnect all the nodes that aren't reserved.
+			for peer_id in self.data.connected_peers(set_id.0).cloned().collect::<Vec<_>>().into_iter() {
+				if self.reserved_nodes[set_id.0].0.contains(&peer_id) {
+					continue;
+				}
+
+				let peer = self.data.peer(set_id.0, &peer_id).into_connected()
+					.expect("We are enumerating connected peers, therefore the peer is connected; qed");
+				peer.disconnect();
+				self.message_queue.push_back(Message::Drop {
+					set_id,
+					peer_id
+				});
+			}
+
+		} else {
 			self.alloc_slots();
 		}
 	}
 
-	fn on_add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
+	/// Adds a node to the given set. The peerset will, if possible and not already the case,
-		self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone());
+	/// try to connect to it.
-		self.data.add_no_slot_node(peer_id);
+	///
-		self.alloc_slots();
+	/// > **Note**: This has the same effect as [`PeersetHandle::add_to_peers_set`].
+	pub fn add_to_peers_set(&mut self, set_id: SetId, peer_id: PeerId) {
+		if let peersstate::Peer::Unknown(entry) = self.data.peer(set_id.0, &peer_id) {
+			entry.discover();
+			self.alloc_slots();
+		}
 	}
 
-	fn on_remove_from_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
+	fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) {
-		if let Some(priority_group) = self.priority_groups.get_mut(group_id) {
+		// Don't do anything if node is reserved.
-			if !priority_group.remove(&peer_id) {
+		if self.reserved_nodes[set_id.0].0.contains(&peer_id) {
-				// `PeerId` wasn't in the group in the first place.
-				return;
-			}
-		} else {
-			// Group doesn't exist, so the `PeerId` can't be in it.
 			return;
 		}
 
-		// If that `PeerId` isn't in any other group, then it is no longer no-slot-occupying.
+		match self.data.peer(set_id.0, &peer_id) {
-		if !self.priority_groups.values().any(|l| l.contains(&peer_id)) {
+			peersstate::Peer::Connected(peer) => {
-			self.data.remove_no_slot_node(&peer_id);
+				self.message_queue.push_back(Message::Drop {
-		}
+					set_id,
-
+					peer_id: peer.peer_id().clone(),
-		// Disconnect the peer if necessary.
+				});
-		if group_id != RESERVED_NODES && self.reserved_only {
+				peer.disconnect().forget_peer();
-			if let peersstate::Peer::Connected(peer) = self.data.peer(&peer_id) {
-				peer.disconnect();
-				self.message_queue.push_back(Message::Drop(peer_id));
 			}
+			peersstate::Peer::NotConnected(peer) => { peer.forget_peer(); }
+			peersstate::Peer::Unknown(_) => {}
 		}
 	}
 
@@ -342,33 +410,29 @@ impl Peerset {
 		// We want reputations to be up-to-date before adjusting them.
 		self.update_time();
 
-		match self.data.peer(&peer_id) {
+		let mut reputation = self.data.peer_reputation(peer_id.clone());
-			peersstate::Peer::Connected(mut peer) => {
+		reputation.add_reputation(change.value);
-				peer.add_reputation(change.value);
+		if reputation.reputation() >= BANNED_THRESHOLD {
-				if peer.reputation() < BANNED_THRESHOLD {
+			trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
-					debug!(target: "peerset", "Report {}: {:+} to {}. Reason: {}, Disconnecting",
+				peer_id, change.value, reputation.reputation(), change.reason
-						peer_id, change.value, peer.reputation(), change.reason
+			);
-					);
+			return;
-					peer.disconnect();
+		}
-					self.message_queue.push_back(Message::Drop(peer_id));
+
-				} else {
+		debug!(target: "peerset", "Report {}: {:+} to {}. Reason: {}, Disconnecting",
-					trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
+			peer_id, change.value, reputation.reputation(), change.reason
-						peer_id, change.value, peer.reputation(), change.reason
+		);
-					);
+
-				}
+		drop(reputation);
-			},
+
-			peersstate::Peer::NotConnected(mut peer) => {
+		for set_index in 0..self.data.num_sets() {
-				trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
+			if let peersstate::Peer::Connected(peer) = self.data.peer(set_index, &peer_id) {
-					peer_id, change.value, peer.reputation(), change.reason
+				let peer = peer.disconnect();
-				);
+				self.message_queue.push_back(Message::Drop {
-				peer.add_reputation(change.value)
+					set_id: SetId(set_index),
-			},
+					peer_id: peer.into_peer_id(),
-			peersstate::Peer::Unknown(peer) => {
+				});
-				trace!(target: "peerset", "Discover {}: {:+}. Reason: {}",
+			}
-					peer_id, change.value, change.reason
-				);
-				peer.discover().add_reputation(change.value)
-			},
 		}
 	}
 
@@ -403,27 +467,35 @@ impl Peerset {
 					}
 					reput.saturating_sub(diff)
 				}
-				match self.data.peer(&peer_id) {
+
-					peersstate::Peer::Connected(mut peer) => {
+				let mut peer_reputation = self.data.peer_reputation(peer_id.clone());
-						let before = peer.reputation();
+
-						let after = reput_tick(before);
+				let before = peer_reputation.reputation();
-						trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
+				let after = reput_tick(before);
-						peer.set_reputation(after)
+				trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
-					}
+				peer_reputation.set_reputation(after);
-					peersstate::Peer::NotConnected(mut peer) => {
+
-						if peer.reputation() == 0 &&
+				if after != 0 {
-							peer.last_connected_or_discovered() + FORGET_AFTER < now
+					continue;
-						{
+				}
-							peer.forget_peer();
+
-						} else {
+				drop(peer_reputation);
-							let before = peer.reputation();
+
-							let after = reput_tick(before);
+				// If the peer reaches a reputation of 0, and there is no connection to it,
-							trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
+				// forget it.
-							peer.set_reputation(after)
+				for set_index in 0..self.data.num_sets() {
+					match self.data.peer(set_index, &peer_id) {
+						peersstate::Peer::Connected(_) => {}
+						peersstate::Peer::NotConnected(peer) => {
+							if peer.last_connected_or_discovered() + FORGET_AFTER < now {
+								peer.forget_peer();
+							}
+						}
+						peersstate::Peer::Unknown(_) => {
+							// Happens if this peer does not belong to this set.
 						}
 					}
-					peersstate::Peer::Unknown(_) => unreachable!("We iterate over known peers; qed")
+				}
-				};
 			}
 		}
 	}
@@ -433,89 +505,54 @@ impl Peerset {
 		self.update_time();
 
 		// Try to connect to all the reserved nodes that we are not connected to.
-		loop {
+		for set_index in 0..self.data.num_sets() {
-			let next = {
+			for reserved_node in &self.reserved_nodes[set_index].0 {
-				let data = &mut self.data;
+				let entry = match self.data.peer(set_index, reserved_node) {
-				self.priority_groups
+					peersstate::Peer::Unknown(n) => n.discover(),
-					.get(RESERVED_NODES)
+					peersstate::Peer::NotConnected(n) => n,
-					.into_iter()
+					peersstate::Peer::Connected(_) => continue,
-					.flatten()
+				};
-					.find(move |n| {
-						data.peer(n).into_connected().is_none()
-					})
-					.cloned()
-			};
 
-			let next = match next {
+				match entry.try_outgoing() {
-				Some(n) => n,
+					Ok(conn) => self.message_queue.push_back(Message::Connect {
-				None => break,
+						set_id: SetId(set_index),
-			};
+						peer_id: conn.into_peer_id()
+					}),
+					Err(_) => {
+						// An error is returned only if no slot is available. Reserved nodes are
+						// marked in the state machine with a flag saying "doesn't occupy a slot",
+						// and as such this should never happen.
+						debug_assert!(false);
+						log::error!(
+							target: "peerset",
+							"Not enough slots to connect to reserved node"
+						);
+					}
+				}
+			}
+		}
 
-			let next = match self.data.peer(&next) {
+		// Now, we try to connect to other nodes.
-				peersstate::Peer::Unknown(n) => n.discover(),
+		for set_index in 0..self.data.num_sets() {
-				peersstate::Peer::NotConnected(n) => n,
+			// Nothing more to do if we're in reserved mode.
-				peersstate::Peer::Connected(_) => {
+			if self.reserved_nodes[set_index].1 {
-					debug_assert!(false, "State inconsistency: not connected state");
+				continue;
+			}
+
+			// Try to grab the next node to attempt to connect to.
+			while let Some(next) = self.data.highest_not_connected_peer(set_index) {
+				// Don't connect to nodes with an abysmal reputation.
+				if next.reputation() < BANNED_THRESHOLD {
 					break;
 				}
-			};
 
-			match next.try_outgoing() {
+				match next.try_outgoing() {
-				Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
+					Ok(conn) => self.message_queue.push_back(Message::Connect {
-				Err(_) => break,	// No more slots available.
+						set_id: SetId(set_index),
-			}
+						peer_id: conn.into_peer_id()
-		}
+					}),
-
+					Err(_) => break,	// No more slots available.
-		// Nothing more to do if we're in reserved mode.
-		if self.reserved_only {
-			return;
-		}
-
-		// Try to connect to all the nodes in priority groups and that we are not connected to.
-		loop {
-			let next = {
-				let data = &mut self.data;
-				self.priority_groups
-					.values()
-					.flatten()
-					.find(move |n| {
-						data.peer(n).into_connected().is_none()
-					})
-					.cloned()
-			};
-
-			let next = match next {
-				Some(n) => n,
-				None => break,
-			};
-
-			let next = match self.data.peer(&next) {
-				peersstate::Peer::Unknown(n) => n.discover(),
-				peersstate::Peer::NotConnected(n) => n,
-				peersstate::Peer::Connected(_) => {
-					debug_assert!(false, "State inconsistency: not connected state");
-					break;
 				}
-			};
-
-			match next.try_outgoing() {
-				Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
-				Err(_) => break,	// No more slots available.
-			}
-		}
-
-		// Now, we try to connect to non-priority nodes.
-		while let Some(next) = self.data.highest_not_connected_peer() {
-			// Don't connect to nodes with an abysmal reputation.
-			if next.reputation() < BANNED_THRESHOLD {
-				break;
-			}
-
-			match next.try_outgoing() {
-				Ok(conn) => self
-					.message_queue
-					.push_back(Message::Connect(conn.into_peer_id())),
-				Err(_) => break, // No more slots available.
 			}
 		}
 	}
@@ -530,16 +567,19 @@ impl Peerset {
 	// Implementation note: because of concurrency issues, it is possible that we push a `Connect`
 	// message to the output channel with a `PeerId`, and that `incoming` gets called with the same
 	// `PeerId` before that message has been read by the user. In this situation we must not answer.
-	pub fn incoming(&mut self, peer_id: PeerId, index: IncomingIndex) {
+	pub fn incoming(&mut self, set_id: SetId, peer_id: PeerId, index: IncomingIndex) {
 		trace!(target: "peerset", "Incoming {:?}", peer_id);
+
 		self.update_time();
 
-		if self.reserved_only && !self.priority_groups.get(RESERVED_NODES).map_or(false, |n| n.contains(&peer_id)) {
+		if self.reserved_nodes[set_id.0].1 {
-			self.message_queue.push_back(Message::Reject(index));
+			if !self.reserved_nodes[set_id.0].0.contains(&peer_id) {
-			return;
+				self.message_queue.push_back(Message::Reject(index));
+				return;
+			}
 		}
 
-		let not_connected = match self.data.peer(&peer_id) {
+		let not_connected = match self.data.peer(set_id.0, &peer_id) {
 			// If we're already connected, don't answer, as the docs mention.
 			peersstate::Peer::Connected(_) => return,
 			peersstate::Peer::NotConnected(mut entry) => {
@@ -564,11 +604,11 @@ impl Peerset {
 	///
 	/// Must only be called after the PSM has either generated a `Connect` message with this
 	/// `PeerId`, or accepted an incoming connection with this `PeerId`.
-	pub fn dropped(&mut self, peer_id: PeerId) {
+	pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId) {
 		// We want reputations to be up-to-date before adjusting them.
 		self.update_time();
 
-		match self.data.peer(&peer_id) {
+		match self.data.peer(set_id.0, &peer_id) {
 			peersstate::Peer::Connected(mut entry) => {
 				// Decrease the node's reputation so that we don't try it again and again and again.
 				entry.add_reputation(DISCONNECT_REPUTATION_CHANGE);
@@ -583,25 +623,6 @@ impl Peerset {
 		self.alloc_slots();
 	}
 
-	/// Adds discovered peer ids to the PSM.
-	///
-	/// > **Note**: There is no equivalent "expired" message, meaning that it is the responsibility
-	/// >           of the PSM to remove `PeerId`s that fail to dial too often.
-	pub fn discovered<I: IntoIterator<Item = PeerId>>(&mut self, peer_ids: I) {
-		let mut discovered_any = false;
-
-		for peer_id in peer_ids {
-			if let peersstate::Peer::Unknown(entry) = self.data.peer(&peer_id) {
-				entry.discover();
-				discovered_any = true;
-			}
-		}
-
-		if discovered_any {
-			self.alloc_slots();
-		}
-	}
-
 	/// Reports an adjustment to the reputation of the given peer.
 	pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) {
 		// We don't immediately perform the adjustments in order to have state consistency. We
@@ -615,23 +636,29 @@ impl Peerset {
 		self.update_time();
 
 		json!({
-			"nodes": self.data.peers().cloned().collect::<Vec<_>>().into_iter().map(|peer_id| {
+			"sets": (0..self.data.num_sets()).map(|set_index| {
-				let state = match self.data.peer(&peer_id) {
+				json!({
-					peersstate::Peer::Connected(entry) => json!({
+					"nodes": self.data.peers().cloned().collect::<Vec<_>>().into_iter().filter_map(|peer_id| {
-						"connected": true,
+						let state = match self.data.peer(set_index, &peer_id) {
-						"reputation": entry.reputation()
+							peersstate::Peer::Connected(entry) => json!({
-					}),
+								"connected": true,
-					peersstate::Peer::NotConnected(entry) => json!({
+								"reputation": entry.reputation()
-						"connected": false,
+							}),
-						"reputation": entry.reputation()
+							peersstate::Peer::NotConnected(entry) => json!({
-					}),
+								"connected": false,
-					peersstate::Peer::Unknown(_) =>
+								"reputation": entry.reputation()
-						unreachable!("We iterate over the known peers; QED")
+							}),
-				};
+							peersstate::Peer::Unknown(_) => return None,
+						};
 
-				(peer_id.to_base58(), state)
+						Some((peer_id.to_base58(), state))
-			}).collect::<HashMap<_, _>>(),
+					}).collect::<HashMap<_, _>>(),
-			"reserved_only": self.reserved_only,
+					"reserved_nodes": self.reserved_nodes[set_index].0.iter().map(|peer_id| {
+						peer_id.to_base58()
+					}).collect::<HashSet<_>>(),
+					"reserved_only": self.reserved_nodes[set_index].1,
+				})
+			}).collect::<Vec<_>>(),
 			"message_queue": self.message_queue.len(),
 		})
 	}
@@ -640,11 +667,6 @@ impl Peerset {
 	pub fn num_discovered_peers(&self) -> usize {
 		self.data.peers().len()
 	}
-
-	/// Returns the content of a priority group.
-	pub fn priority_group(&self, group_id: &str) -> Option<impl ExactSizeIterator<Item = &PeerId>> {
-		self.priority_groups.get(group_id).map(|l| l.iter())
-	}
 }
 
 impl Stream for Peerset {
@@ -663,22 +685,20 @@ impl Stream for Peerset {
 			};
 
 			match action {
-				Action::AddReservedPeer(peer_id) =>
+				Action::AddReservedPeer(set_id, peer_id) =>
-					self.on_add_reserved_peer(peer_id),
+					self.on_add_reserved_peer(set_id, peer_id),
-				Action::RemoveReservedPeer(peer_id) =>
+				Action::RemoveReservedPeer(set_id, peer_id) =>
-					self.on_remove_reserved_peer(peer_id),
+					self.on_remove_reserved_peer(set_id, peer_id),
-				Action::SetReservedPeers(peer_ids) =>
+				Action::SetReservedPeers(set_id, peer_ids) =>
-					self.on_set_reserved_peers(peer_ids),
+					self.on_set_reserved_peers(set_id, peer_ids),
-				Action::SetReservedOnly(reserved) =>
+				Action::SetReservedOnly(set_id, reserved) =>
-					self.on_set_reserved_only(reserved),
+					self.on_set_reserved_only(set_id, reserved),
 				Action::ReportPeer(peer_id, score_diff) =>
 					self.on_report_peer(peer_id, score_diff),
-				Action::SetPriorityGroup(group_id, peers) =>
+				Action::AddToPeersSet(sets_name, peer_id) =>
-					self.on_set_priority_group(&group_id, peers),
+					self.add_to_peers_set(sets_name, peer_id),
-				Action::AddToPriorityGroup(group_id, peer_id) =>
+				Action::RemoveFromPeersSet(sets_name, peer_id) =>
-					self.on_add_to_priority_group(&group_id, peer_id),
+					self.on_remove_from_peers_set(sets_name, peer_id),
-				Action::RemoveFromPriorityGroup(group_id, peer_id) =>
-					self.on_remove_from_priority_group(&group_id, peer_id),
 			}
 		}
 	}
@@ -688,7 +708,7 @@ impl Stream for Peerset {
 mod tests {
 	use libp2p::PeerId;
 	use futures::prelude::*;
-	use super::{PeersetConfig, Peerset, Message, IncomingIndex, ReputationChange, BANNED_THRESHOLD};
+	use super::{PeersetConfig, Peerset, Message, IncomingIndex, ReputationChange, SetConfig, SetId, BANNED_THRESHOLD};
 	use std::{pin::Pin, task::Poll, thread, time::Duration};
 
 	fn assert_messages(mut peerset: Peerset, messages: Vec<Message>) -> Peerset {
@@ -712,20 +732,22 @@ mod tests {
 		let reserved_peer = PeerId::random();
 		let reserved_peer2 = PeerId::random();
 		let config = PeersetConfig {
-			in_peers: 0,
+			sets: vec![SetConfig {
-			out_peers: 2,
+				in_peers: 0,
-			bootnodes: vec![bootnode],
+				out_peers: 2,
-			reserved_only: true,
+				bootnodes: vec![bootnode],
-			priority_groups: Vec::new(),
+				reserved_nodes: Default::default(),
+				reserved_only: true,
+			}],
 		};
 
 		let (peerset, handle) = Peerset::from_config(config);
-		handle.add_reserved_peer(reserved_peer.clone());
+		handle.add_reserved_peer(SetId::from(0), reserved_peer.clone());
-		handle.add_reserved_peer(reserved_peer2.clone());
+		handle.add_reserved_peer(SetId::from(0), reserved_peer2.clone());
 
 		assert_messages(peerset, vec![
-			Message::Connect(reserved_peer),
+			Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer },
-			Message::Connect(reserved_peer2)
+			Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer2 }
 		]);
 	}
 
@@ -740,21 +762,23 @@ mod tests {
 		let ii3 = IncomingIndex(3);
 		let ii4 = IncomingIndex(3);
 		let config = PeersetConfig {
-			in_peers: 2,
+			sets: vec![SetConfig {
-			out_peers: 1,
+				in_peers: 2,
-			bootnodes: vec![bootnode.clone()],
+				out_peers: 1,
-			reserved_only: false,
+				bootnodes: vec![bootnode.clone()],
-			priority_groups: Vec::new(),
+				reserved_nodes: Default::default(),
+				reserved_only: false,
+			}],
 		};
 
 		let (mut peerset, _handle) = Peerset::from_config(config);
-		peerset.incoming(incoming.clone(), ii);
+		peerset.incoming(SetId::from(0), incoming.clone(), ii);
-		peerset.incoming(incoming, ii4);
+		peerset.incoming(SetId::from(0), incoming.clone(), ii4);
-		peerset.incoming(incoming2, ii2);
+		peerset.incoming(SetId::from(0), incoming2.clone(), ii2);
-		peerset.incoming(incoming3, ii3);
+		peerset.incoming(SetId::from(0), incoming3.clone(), ii3);
 
 		assert_messages(peerset, vec![
-			Message::Connect(bootnode),
+			Message::Connect { set_id: SetId::from(0), peer_id: bootnode.clone() },
 			Message::Accept(ii),
 			Message::Accept(ii2),
 			Message::Reject(ii3),
@@ -766,15 +790,17 @@ mod tests {
 		let incoming = PeerId::random();
 		let ii = IncomingIndex(1);
 		let config = PeersetConfig {
-			in_peers: 50,
+			sets: vec![SetConfig {
-			out_peers: 50,
+				in_peers: 50,
-			bootnodes: vec![],
+				out_peers: 50,
-			reserved_only: true,
+				bootnodes: vec![],
-			priority_groups: vec![],
+				reserved_nodes: Default::default(),
+				reserved_only: true,
+			}],
 		};
 
 		let (mut peerset, _) = Peerset::from_config(config);
-		peerset.incoming(incoming, ii);
+		peerset.incoming(SetId::from(0), incoming.clone(), ii);
 
 		assert_messages(peerset, vec![
 			Message::Reject(ii),
@@ -787,32 +813,36 @@ mod tests {
 		let discovered = PeerId::random();
 		let discovered2 = PeerId::random();
 		let config = PeersetConfig {
-			in_peers: 0,
+			sets: vec![SetConfig {
-			out_peers: 2,
+				in_peers: 0,
-			bootnodes: vec![bootnode.clone()],
+				out_peers: 2,
-			reserved_only: false,
+				bootnodes: vec![bootnode.clone()],
-			priority_groups: vec![],
+				reserved_nodes: Default::default(),
+				reserved_only: false,
+			}],
 		};
 
 		let (mut peerset, _handle) = Peerset::from_config(config);
-		peerset.discovered(Some(discovered.clone()));
+		peerset.add_to_peers_set(SetId::from(0), discovered.clone());
-		peerset.discovered(Some(discovered.clone()));
+		peerset.add_to_peers_set(SetId::from(0), discovered.clone());
-		peerset.discovered(Some(discovered2));
+		peerset.add_to_peers_set(SetId::from(0), discovered2);
 
 		assert_messages(peerset, vec![
-			Message::Connect(bootnode),
+			Message::Connect { set_id: SetId::from(0), peer_id: bootnode },
-			Message::Connect(discovered),
+			Message::Connect { set_id: SetId::from(0), peer_id: discovered },
 		]);
 	}
 
 	#[test]
 	fn test_peerset_banned() {
 		let (mut peerset, handle) = Peerset::from_config(PeersetConfig {
-			in_peers: 25,
+			sets: vec![SetConfig {
-			out_peers: 25,
+				in_peers: 25,
-			bootnodes: vec![],
+				out_peers: 25,
-			reserved_only: false,
+				bootnodes: vec![],
-			priority_groups: vec![],
+				reserved_nodes: Default::default(),
+				reserved_only: false,
+			}],
 		});
 
 		// We ban a node by setting its reputation under the threshold.
@@ -824,7 +854,7 @@ mod tests {
 			assert_eq!(Stream::poll_next(Pin::new(&mut peerset), cx), Poll::Pending);
 
 			// Check that an incoming connection from that node gets refused.
-			peerset.incoming(peer_id.clone(), IncomingIndex(1));
+			peerset.incoming(SetId::from(0), peer_id.clone(), IncomingIndex(1));
 			if let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
 				assert_eq!(msg.unwrap(), Message::Reject(IncomingIndex(1)));
 			} else {
@@ -835,7 +865,7 @@ mod tests {
 			thread::sleep(Duration::from_millis(1500));
 
 			// Try again. This time the node should be accepted.
-			peerset.incoming(peer_id.clone(), IncomingIndex(2));
+			peerset.incoming(SetId::from(0), peer_id.clone(), IncomingIndex(2));
 			while let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
 				assert_eq!(msg.unwrap(), Message::Accept(IncomingIndex(2)));
 			}

substrate/client/peerset/src/peersstate.rs

@@ -19,9 +19,10 @@
 //! Reputation and slots allocation system behind the peerset.
 //!
 //! The [`PeersState`] state machine is responsible for managing the reputation and allocating
-//! slots. It holds a list of nodes, each associated with a reputation value and whether we are
+//! slots. It holds a list of nodes, each associated with a reputation value, a list of sets the
-//! connected or not to this node. Thanks to this list, it knows how many slots are occupied. It
+//! node belongs to, and for each set whether we are connected or not to this node. Thanks to this
-//! also holds a list of nodes which don't occupy slots.
+//! list, it knows how many slots are occupied. It also holds a list of nodes which don't occupy
+//! slots.
 //!
 //! > Note: This module is purely dedicated to managing slots and reputations. Features such as
 //! >       for example connecting to some nodes in priority should be added outside of this
@@ -29,7 +30,10 @@
 
 use libp2p::PeerId;
 use log::error;
-use std::{borrow::Cow, collections::{HashSet, HashMap}};
+use std::{
+	borrow::Cow,
+	collections::{HashMap, HashSet, hash_map::{Entry, OccupiedEntry}},
+};
 use wasm_timer::Instant;
 
 /// State storage behind the peerset.
@@ -48,16 +52,33 @@ pub struct PeersState {
 	///           sort, to make the logic easier.
 	nodes: HashMap<PeerId, Node>,
 
-	/// Number of slot-occupying nodes for which the `ConnectionState` is `In`.
+	/// Configuration of each set. The size of this `Vec` is never modified.
+	sets: Vec<SetInfo>,
+}
+
+/// Configuration of a single set.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub struct SetConfig {
+	/// Maximum allowed number of slot-occupying nodes for ingoing connections.
+	pub in_peers: u32,
+
+	/// Maximum allowed number of slot-occupying nodes for outgoing connections.
+	pub out_peers: u32,
+}
+
+/// State of a single set.
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct SetInfo {
+	/// Number of slot-occupying nodes for which the `MembershipState` is `In`.
 	num_in: u32,
 
-	/// Number of slot-occupying nodes for which the `ConnectionState` is `In`.
+	/// Number of slot-occupying nodes for which the `MembershipState` is `In`.
 	num_out: u32,
 
-	/// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `In`.
+	/// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `In`.
 	max_in: u32,
 
-	/// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `Out`.
+	/// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `Out`.
 	max_out: u32,
 
 	/// List of node identities (discovered or not) that don't occupy slots.
@@ -69,35 +90,37 @@ pub struct PeersState {
 }
 
 /// State of a single node that we know about.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 struct Node {
-	/// Whether we are connected to this node.
+	/// List of sets the node belongs to.
-	connection_state: ConnectionState,
+	/// Always has a fixed size equal to the one of [`PeersState::set`]. The various possible sets
+	/// are indices into this `Vec`.
+	sets: Vec<MembershipState>,
 
 	/// Reputation value of the node, between `i32::min_value` (we hate that node) and
 	/// `i32::max_value` (we love that node).
 	reputation: i32,
 }
 
-impl Default for Node {
+impl Node {
-	fn default() -> Node {
+	fn new(num_sets: usize) -> Node {
 		Node {
-			connection_state: ConnectionState::NotConnected {
+			sets: (0..num_sets).map(|_| MembershipState::NotMember).collect(),
-				last_connected: Instant::now(),
-			},
 			reputation: 0,
 		}
 	}
 }
 
-/// Whether we are connected to a node.
+/// Whether we are connected to a node in the context of a specific set.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum ConnectionState {
+enum MembershipState {
+	/// Node isn't part of that set.
+	NotMember,
 	/// We are connected through an ingoing connection.
 	In,
 	/// We are connected through an outgoing connection.
 	Out,
-	/// We are not connected to this node.
+	/// Node is part of that set, but we are not connected to it.
 	NotConnected {
 		/// When we were last connected to the node, or if we were never connected when we
 		/// discovered it.
@@ -105,50 +128,87 @@ enum ConnectionState {
 	},
 }
 
-impl ConnectionState {
+impl MembershipState {
 	/// Returns `true` for `In` and `Out`.
 	fn is_connected(self) -> bool {
 		match self {
-			ConnectionState::In => true,
+			MembershipState::NotMember => false,
-			ConnectionState::Out => true,
+			MembershipState::In => true,
-			ConnectionState::NotConnected { .. } => false,
+			MembershipState::Out => true,
+			MembershipState::NotConnected { .. } => false,
 		}
 	}
 }
 
 impl PeersState {
 	/// Builds a new empty `PeersState`.
-	pub fn new(in_peers: u32, out_peers: u32) -> Self {
+	pub fn new(sets: impl IntoIterator<Item = SetConfig>) -> Self {
 		PeersState {
 			nodes: HashMap::new(),
-			num_in: 0,
+			sets: sets
-			num_out: 0,
+				.into_iter()
-			max_in: in_peers,
+				.map(|config| SetInfo {
-			max_out: out_peers,
+					num_in: 0,
-			no_slot_nodes: HashSet::new(),
+					num_out: 0,
+					max_in: config.in_peers,
+					max_out: config.out_peers,
+					no_slot_nodes: HashSet::new(),
+				})
+				.collect(),
 		}
 	}
 
-	/// Returns an object that grants access to the state of a peer.
+	/// Returns the number of sets.
-	pub fn peer<'a>(&'a mut self, peer_id: &'a PeerId) -> Peer<'a> {
+	///
-		match self.nodes.get_mut(peer_id) {
+	/// Corresponds to the number of elements passed to [`PeersState::new`].
-			None => Peer::Unknown(UnknownPeer {
+	pub fn num_sets(&self) -> usize {
+		self.sets.len()
+	}
+
+	/// Returns an object that grants access to the reputation value of a peer.
+	pub fn peer_reputation(&mut self, peer_id: PeerId) -> Reputation {
+		if !self.nodes.contains_key(&peer_id) {
+			self.nodes.insert(peer_id.clone(), Node::new(self.sets.len()));
+		}
+
+		let entry = match self.nodes.entry(peer_id) {
+			Entry::Vacant(_) => unreachable!("guaranteed to be inserted above; qed"),
+			Entry::Occupied(e) => e,
+		};
+
+		Reputation { node: Some(entry) }
+	}
+
+	/// Returns an object that grants access to the state of a peer in the context of a specific
+	/// set.
+	///
+	/// # Panic
+	///
+	/// `set` must be within range of the sets passed to [`PeersState::new`].
+	///
+	pub fn peer<'a>(&'a mut self, set: usize, peer_id: &'a PeerId) -> Peer<'a> {
+		// The code below will panic anyway if this happens to be false, but this earlier assert
+		// makes it explicit what is wrong.
+		assert!(set < self.sets.len());
+
+		match self.nodes.get_mut(peer_id).map(|p| &p.sets[set]) {
+			None | Some(MembershipState::NotMember) => Peer::Unknown(UnknownPeer {
 				parent: self,
+				set,
 				peer_id: Cow::Borrowed(peer_id),
 			}),
-			Some(peer) => {
+			Some(MembershipState::In) | Some(MembershipState::Out) => {
-				if peer.connection_state.is_connected() {
+				Peer::Connected(ConnectedPeer {
-					Peer::Connected(ConnectedPeer {
+					state: self,
-						state: self,
+					set,
-						peer_id: Cow::Borrowed(peer_id),
+					peer_id: Cow::Borrowed(peer_id),
-					})
+				})
-				} else {
-					Peer::NotConnected(NotConnectedPeer {
-						state: self,
-						peer_id: Cow::Borrowed(peer_id),
-					})
-				}
 			}
+			Some(MembershipState::NotConnected { .. }) => Peer::NotConnected(NotConnectedPeer {
+				state: self,
+				set,
+				peer_id: Cow::Borrowed(peer_id),
+			}),
 		}
 	}
 
@@ -159,22 +219,49 @@ impl PeersState {
 		self.nodes.keys()
 	}
 
-	/// Returns the list of peers we are connected to.
+	/// Returns the list of peers we are connected to in the context of a specific set.
+	///
+	/// # Panic
+	///
+	/// `set` must be within range of the sets passed to [`PeersState::new`].
+	///
 	// Note: this method could theoretically return a `ConnectedPeer`, but implementing that
 	// isn't simple.
-	pub fn connected_peers(&self) -> impl Iterator<Item = &PeerId> {
+	pub fn connected_peers(&self, set: usize) -> impl Iterator<Item = &PeerId> {
-		self.nodes.iter()
+		// The code below will panic anyway if this happens to be false, but this earlier assert
-			.filter(|(_, p)| p.connection_state.is_connected())
+		// makes it explicit what is wrong.
+		assert!(set < self.sets.len());
+
+		self.nodes
+			.iter()
+			.filter(move |(_, p)| p.sets[set].is_connected())
 			.map(|(p, _)| p)
 	}
 
 	/// Returns the peer with the highest reputation and that we are not connected to.
 	///
 	/// If multiple nodes have the same reputation, which one is returned is unspecified.
-	pub fn highest_not_connected_peer(&mut self) -> Option<NotConnectedPeer> {
+	///
-		let outcome = self.nodes
+	/// # Panic
+	///
+	/// `set` must be within range of the sets passed to [`PeersState::new`].
+	///
+	pub fn highest_not_connected_peer(&mut self, set: usize) -> Option<NotConnectedPeer> {
+		// The code below will panic anyway if this happens to be false, but this earlier assert
+		// makes it explicit what is wrong.
+		assert!(set < self.sets.len());
+
+		let outcome = self
+			.nodes
 			.iter_mut()
-			.filter(|(_, Node { connection_state, .. })| !connection_state.is_connected())
+			.filter(|(_, Node { sets, .. })| {
+				match sets[set] {
+					MembershipState::NotMember => false,
+					MembershipState::In => false,
+					MembershipState::Out => false,
+					MembershipState::NotConnected { .. } => true,
+				}
+			})
 			.fold(None::<(&PeerId, &mut Node)>, |mut cur_node, to_try| {
 				if let Some(cur_node) = cur_node.take() {
 					if cur_node.1.reputation >= to_try.1.reputation {
@@ -188,6 +275,7 @@ impl PeersState {
 		if let Some(peer_id) = outcome {
 			Some(NotConnectedPeer {
 				state: self,
+				set,
 				peer_id: Cow::Owned(peer_id),
 			})
 		} else {
@@ -197,48 +285,48 @@ impl PeersState {
 
 	/// Add a node to the list of nodes that don't occupy slots.
 	///
-	/// Has no effect if the peer was already in the group.
+	/// Has no effect if the node was already in the group.
-	pub fn add_no_slot_node(&mut self, peer_id: PeerId) {
+	pub fn add_no_slot_node(&mut self, set: usize, peer_id: PeerId) {
 		// Reminder: `HashSet::insert` returns false if the node was already in the set
-		if !self.no_slot_nodes.insert(peer_id.clone()) {
+		if !self.sets[set].no_slot_nodes.insert(peer_id.clone()) {
 			return;
 		}
 
 		if let Some(peer) = self.nodes.get_mut(&peer_id) {
-			match peer.connection_state {
+			match peer.sets[set] {
-				ConnectionState::In => self.num_in -= 1,
+				MembershipState::In => self.sets[set].num_in -= 1,
-				ConnectionState::Out => self.num_out -= 1,
+				MembershipState::Out => self.sets[set].num_out -= 1,
-				ConnectionState::NotConnected { .. } => {},
+				MembershipState::NotConnected { .. } | MembershipState::NotMember => {}
 			}
 		}
 	}
 
 	/// Removes a node from the list of nodes that don't occupy slots.
 	///
-	/// Has no effect if the peer was not in the group.
+	/// Has no effect if the node was not in the group.
-	pub fn remove_no_slot_node(&mut self, peer_id: &PeerId) {
+	pub fn remove_no_slot_node(&mut self, set: usize, peer_id: &PeerId) {
 		// Reminder: `HashSet::remove` returns false if the node was already not in the set
-		if !self.no_slot_nodes.remove(peer_id) {
+		if !self.sets[set].no_slot_nodes.remove(peer_id) {
 			return;
 		}
 
 		if let Some(peer) = self.nodes.get_mut(peer_id) {
-			match peer.connection_state {
+			match peer.sets[set] {
-				ConnectionState::In => self.num_in += 1,
+				MembershipState::In => self.sets[set].num_in += 1,
-				ConnectionState::Out => self.num_out += 1,
+				MembershipState::Out => self.sets[set].num_out += 1,
-				ConnectionState::NotConnected { .. } => {},
+				MembershipState::NotConnected { .. } | MembershipState::NotMember => {}
 			}
 		}
 	}
 }
 
-/// Grants access to the state of a peer in the `PeersState`.
+/// Grants access to the state of a peer in the [`PeersState`] in the context of a specific set.
 pub enum Peer<'a> {
 	/// We are connected to this node.
 	Connected(ConnectedPeer<'a>),
 	/// We are not connected to this node.
 	NotConnected(NotConnectedPeer<'a>),
-	/// We have never heard of this node.
+	/// We have never heard of this node, or it is not part of the set.
 	Unknown(UnknownPeer<'a>),
 }
 
@@ -255,7 +343,7 @@ impl<'a> Peer<'a> {
 
 	/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
 	/// otherwise.
-	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
+	#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
 	pub fn into_not_connected(self) -> Option<NotConnectedPeer<'a>> {
 		match self {
 			Peer::Connected(_) => None,
@@ -266,7 +354,7 @@ impl<'a> Peer<'a> {
 
 	/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
 	/// otherwise.
-	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
+	#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
 	pub fn into_unknown(self) -> Option<UnknownPeer<'a>> {
 		match self {
 			Peer::Connected(_) => None,
@@ -279,10 +367,16 @@ impl<'a> Peer<'a> {
 /// A peer that is connected to us.
 pub struct ConnectedPeer<'a> {
 	state: &'a mut PeersState,
+	set: usize,
 	peer_id: Cow<'a, PeerId>,
 }
 
 impl<'a> ConnectedPeer<'a> {
+	/// Get the `PeerId` associated to this `ConnectedPeer`.
+	pub fn peer_id(&self) -> &PeerId {
+		&self.peer_id
+	}
+
 	/// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it.
 	pub fn into_peer_id(self) -> PeerId {
 		self.peer_id.into_owned()
@@ -290,65 +384,74 @@ impl<'a> ConnectedPeer<'a> {
 
 	/// Switches the peer to "not connected".
 	pub fn disconnect(self) -> NotConnectedPeer<'a> {
-		let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
+		let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
-		if let Some(mut node) = self.state.nodes.get_mut(&*self.peer_id) {
+		if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
 			if !is_no_slot_occupy {
-				match node.connection_state {
+				match node.sets[self.set] {
-					ConnectionState::In => self.state.num_in -= 1,
+					MembershipState::In => self.state.sets[self.set].num_in -= 1,
-					ConnectionState::Out => self.state.num_out -= 1,
+					MembershipState::Out => self.state.sets[self.set].num_out -= 1,
-					ConnectionState::NotConnected { .. } =>
+					MembershipState::NotMember | MembershipState::NotConnected { .. } => {
-						debug_assert!(false, "State inconsistency: disconnecting a disconnected node")
+						debug_assert!(
+							false,
+							"State inconsistency: disconnecting a disconnected node"
+						)
+					}
 				}
 			}
-			node.connection_state = ConnectionState::NotConnected {
+			node.sets[self.set] = MembershipState::NotConnected {
 				last_connected: Instant::now(),
 			};
 		} else {
-			debug_assert!(false, "State inconsistency: disconnecting a disconnected node");
+			debug_assert!(
+				false,
+				"State inconsistency: disconnecting a disconnected node"
+			);
 		}
 
 		NotConnectedPeer {
 			state: self.state,
+			set: self.set,
 			peer_id: self.peer_id,
 		}
 	}
 
-	/// Returns the reputation value of the node.
-	pub fn reputation(&self) -> i32 {
-		self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation)
-	}
-
-	/// Sets the reputation of the peer.
-	pub fn set_reputation(&mut self, value: i32) {
-		if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
-			node.reputation = value;
-		} else {
-			debug_assert!(false, "State inconsistency: set_reputation on an unknown node");
-		}
-	}
-
 	/// Performs an arithmetic addition on the reputation score of that peer.
 	///
 	/// In case of overflow, the value will be capped.
+	///
+	/// > **Note**: Reputation values aren't specific to a set but are global per peer.
 	pub fn add_reputation(&mut self, modifier: i32) {
 		if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
 			node.reputation = node.reputation.saturating_add(modifier);
 		} else {
-			debug_assert!(false, "State inconsistency: add_reputation on an unknown node");
+			debug_assert!(
+				false,
+				"State inconsistency: add_reputation on an unknown node"
+			);
 		}
 	}
+
+	/// Returns the reputation value of the node.
+	///
+	/// > **Note**: Reputation values aren't specific to a set but are global per peer.
+	pub fn reputation(&self) -> i32 {
+		self.state
+			.nodes
+			.get(&*self.peer_id)
+			.map_or(0, |p| p.reputation)
+	}
 }
 
 /// A peer that is not connected to us.
 #[derive(Debug)]
 pub struct NotConnectedPeer<'a> {
 	state: &'a mut PeersState,
+	set: usize,
 	peer_id: Cow<'a, PeerId>,
 }
 
 impl<'a> NotConnectedPeer<'a> {
 	/// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it.
-	#[cfg(test)]	// Feel free to remove this if this function is needed outside of tests
 	pub fn into_peer_id(self) -> PeerId {
 		self.peer_id.into_owned()
 	}
@@ -361,7 +464,7 @@ impl<'a> NotConnectedPeer<'a> {
 			None => return,
 		};
 
-		if let ConnectionState::NotConnected { last_connected } = &mut state.connection_state {
+		if let MembershipState::NotConnected { last_connected } = &mut state.sets[self.set] {
 			*last_connected = Instant::now();
 		}
 	}
@@ -383,8 +486,8 @@ impl<'a> NotConnectedPeer<'a> {
 			}
 		};
 
-		match state.connection_state {
+		match state.sets[self.set] {
-			ConnectionState::NotConnected { last_connected } => last_connected,
+			MembershipState::NotConnected { last_connected } => last_connected,
 			_ => {
 				error!(target: "peerset", "State inconsistency with {}", self.peer_id);
 				Instant::now()
@@ -399,25 +502,31 @@ impl<'a> NotConnectedPeer<'a> {
 	///
 	/// Non-slot-occupying nodes don't count towards the number of slots.
 	pub fn try_outgoing(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
-		let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
+		let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
 
 		// Note that it is possible for num_out to be strictly superior to the max, in case we were
 		// connected to reserved node then marked them as not reserved.
-		if self.state.num_out >= self.state.max_out && !is_no_slot_occupy {
+		if self.state.sets[self.set].num_out >= self.state.sets[self.set].max_out
+			&& !is_no_slot_occupy
+		{
 			return Err(self);
 		}
 
-		if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) {
+		if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
-			peer.connection_state = ConnectionState::Out;
+			peer.sets[self.set] = MembershipState::Out;
 			if !is_no_slot_occupy {
-				self.state.num_out += 1;
+				self.state.sets[self.set].num_out += 1;
 			}
 		} else {
-			debug_assert!(false, "State inconsistency: try_outgoing on an unknown node");
+			debug_assert!(
+				false,
+				"State inconsistency: try_outgoing on an unknown node"
+			);
 		}
 
 		Ok(ConnectedPeer {
 			state: self.state,
+			set: self.set,
 			peer_id: self.peer_id,
 		})
 	}
@@ -429,74 +538,95 @@ impl<'a> NotConnectedPeer<'a> {
 	///
 	/// Non-slot-occupying nodes don't count towards the number of slots.
 	pub fn try_accept_incoming(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
-		let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
+		let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
 
 		// Note that it is possible for num_in to be strictly superior to the max, in case we were
 		// connected to reserved node then marked them as not reserved.
-		if self.state.num_in >= self.state.max_in && !is_no_slot_occupy {
+		if self.state.sets[self.set].num_in >= self.state.sets[self.set].max_in
+			&& !is_no_slot_occupy
+		{
 			return Err(self);
 		}
 
-		if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) {
+		if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
-			peer.connection_state = ConnectionState::In;
+			peer.sets[self.set] = MembershipState::In;
 			if !is_no_slot_occupy {
-				self.state.num_in += 1;
+				self.state.sets[self.set].num_in += 1;
 			}
 		} else {
-			debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node");
+			debug_assert!(
+				false,
+				"State inconsistency: try_accept_incoming on an unknown node"
+			);
 		}
 
 		Ok(ConnectedPeer {
 			state: self.state,
+			set: self.set,
 			peer_id: self.peer_id,
 		})
 	}
 
 	/// Returns the reputation value of the node.
+	///
+	/// > **Note**: Reputation values aren't specific to a set but are global per peer.
 	pub fn reputation(&self) -> i32 {
-		self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation)
+		self.state
+			.nodes
+			.get(&*self.peer_id)
+			.map_or(0, |p| p.reputation)
 	}
 
 	/// Sets the reputation of the peer.
+	///
+	/// > **Note**: Reputation values aren't specific to a set but are global per peer.
+	#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
 	pub fn set_reputation(&mut self, value: i32) {
 		if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
 			node.reputation = value;
 		} else {
-			debug_assert!(false, "State inconsistency: set_reputation on an unknown node");
+			debug_assert!(
+				false,
+				"State inconsistency: set_reputation on an unknown node"
+			);
 		}
 	}
 
-	/// Performs an arithmetic addition on the reputation score of that peer.
+	/// Removes the peer from the list of members of the set.
-	///
-	/// In case of overflow, the value will be capped.
-	pub fn add_reputation(&mut self, modifier: i32) {
-		if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
-			node.reputation = node.reputation.saturating_add(modifier);
-		} else {
-			debug_assert!(false, "State inconsistency: add_reputation on an unknown node");
-		}
-	}
-
-	/// Un-discovers the peer. Removes it from the list.
 	pub fn forget_peer(self) -> UnknownPeer<'a> {
-		if self.state.nodes.remove(&*self.peer_id).is_none() {
+		if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
+			debug_assert!(!matches!(peer.sets[self.set], MembershipState::NotMember));
+			peer.sets[self.set] = MembershipState::NotMember;
+
+			// Remove the peer from `self.state.nodes` entirely if it isn't a member of any set.
+			if peer.reputation == 0 && peer
+				.sets
+				.iter()
+				.all(|set| matches!(set, MembershipState::NotMember))
+			{
+				self.state.nodes.remove(&*self.peer_id);
+			}
+		} else {
+			debug_assert!(false, "State inconsistency: forget_peer on an unknown node");
 			error!(
 				target: "peerset",
 				"State inconsistency with {} when forgetting peer",
 				self.peer_id
 			);
-		}
+		};
 
 		UnknownPeer {
 			parent: self.state,
+			set: self.set,
 			peer_id: self.peer_id,
 		}
 	}
 }
 
-/// A peer that we have never heard of.
+/// A peer that we have never heard of or that isn't part of the set.
 pub struct UnknownPeer<'a> {
 	parent: &'a mut PeersState,
+	set: usize,
 	peer_id: Cow<'a, PeerId>,
 }
 
@@ -506,96 +636,240 @@ impl<'a> UnknownPeer<'a> {
 	/// The node starts with a reputation of 0. You can adjust these default
 	/// values using the `NotConnectedPeer` that this method returns.
 	pub fn discover(self) -> NotConnectedPeer<'a> {
-		self.parent.nodes.insert(self.peer_id.clone().into_owned(), Node {
+		let num_sets = self.parent.sets.len();
-			connection_state: ConnectionState::NotConnected {
+
-				last_connected: Instant::now(),
+		self.parent
-			},
+			.nodes
-			reputation: 0,
+			.entry(self.peer_id.clone().into_owned())
-		});
+			.or_insert_with(|| Node::new(num_sets))
+			.sets[self.set] = MembershipState::NotConnected {
+			last_connected: Instant::now(),
+		};
 
-		let state = self.parent;
 		NotConnectedPeer {
-			state,
+			state: self.parent,
+			set: self.set,
 			peer_id: self.peer_id,
 		}
 	}
 }
 
+/// Access to the reputation of a peer.
+pub struct Reputation<'a> {
+	/// Node entry in [`PeersState::nodes`]. Always `Some` except right before dropping.
+	node: Option<OccupiedEntry<'a, PeerId, Node>>,
+}
+
+impl<'a> Reputation<'a> {
+	/// Returns the reputation value of the node.
+	pub fn reputation(&self) -> i32 {
+		self.node.as_ref().unwrap().get().reputation
+	}
+
+	/// Sets the reputation of the peer.
+	pub fn set_reputation(&mut self, value: i32) {
+		self.node.as_mut().unwrap().get_mut().reputation = value;
+	}
+
+	/// Performs an arithmetic addition on the reputation score of that peer.
+	///
+	/// In case of overflow, the value will be capped.
+	pub fn add_reputation(&mut self, modifier: i32) {
+		let reputation = &mut self.node.as_mut().unwrap().get_mut().reputation;
+		*reputation = reputation.saturating_add(modifier);
+	}
+}
+
+impl<'a> Drop for Reputation<'a> {
+	fn drop(&mut self) {
+		if let Some(node) = self.node.take() {
+			if node.get().reputation == 0 &&
+				node.get().sets.iter().all(|set| matches!(set, MembershipState::NotMember))
+			{
+				node.remove();
+			}
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
-	use super::{PeersState, Peer};
+	use super::{Peer, PeersState, SetConfig};
 	use libp2p::PeerId;
+	use std::iter;
 
 	#[test]
 	fn full_slots_in() {
-		let mut peers_state = PeersState::new(1, 1);
+		let mut peers_state = PeersState::new(iter::once(SetConfig {
+			in_peers: 1,
+			out_peers: 1,
+		}));
 		let id1 = PeerId::random();
 		let id2 = PeerId::random();
 
-		if let Peer::Unknown(e) = peers_state.peer(&id1) {
+		if let Peer::Unknown(e) = peers_state.peer(0, &id1) {
 			assert!(e.discover().try_accept_incoming().is_ok());
 		}
 
-		if let Peer::Unknown(e) = peers_state.peer(&id2) {
+		if let Peer::Unknown(e) = peers_state.peer(0, &id2) {
 			assert!(e.discover().try_accept_incoming().is_err());
 		}
 	}
 
 	#[test]
 	fn no_slot_node_doesnt_use_slot() {
-		let mut peers_state = PeersState::new(1, 1);
+		let mut peers_state = PeersState::new(iter::once(SetConfig {
+			in_peers: 1,
+			out_peers: 1,
+		}));
 		let id1 = PeerId::random();
 		let id2 = PeerId::random();
 
-		peers_state.add_no_slot_node(id1.clone());
+		peers_state.add_no_slot_node(0, id1.clone());
-		if let Peer::Unknown(p) = peers_state.peer(&id1) {
+		if let Peer::Unknown(p) = peers_state.peer(0, &id1) {
 			assert!(p.discover().try_accept_incoming().is_ok());
-		} else { panic!() }
+		} else {
+			panic!()
+		}
 
-		if let Peer::Unknown(e) = peers_state.peer(&id2) {
+		if let Peer::Unknown(e) = peers_state.peer(0, &id2) {
 			assert!(e.discover().try_accept_incoming().is_ok());
-		} else { panic!() }
+		} else {
+			panic!()
+		}
 	}
 
 	#[test]
 	fn disconnecting_frees_slot() {
-		let mut peers_state = PeersState::new(1, 1);
+		let mut peers_state = PeersState::new(iter::once(SetConfig {
+			in_peers: 1,
+			out_peers: 1,
+		}));
 		let id1 = PeerId::random();
 		let id2 = PeerId::random();
 
-		assert!(peers_state.peer(&id1).into_unknown().unwrap().discover().try_accept_incoming().is_ok());
+		assert!(peers_state
-		assert!(peers_state.peer(&id2).into_unknown().unwrap().discover().try_accept_incoming().is_err());
+			.peer(0, &id1)
-		peers_state.peer(&id1).into_connected().unwrap().disconnect();
+			.into_unknown()
-		assert!(peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().is_ok());
+			.unwrap()
+			.discover()
+			.try_accept_incoming()
+			.is_ok());
+		assert!(peers_state
+			.peer(0, &id2)
+			.into_unknown()
+			.unwrap()
+			.discover()
+			.try_accept_incoming()
+			.is_err());
+		peers_state
+			.peer(0, &id1)
+			.into_connected()
+			.unwrap()
+			.disconnect();
+		assert!(peers_state
+			.peer(0, &id2)
+			.into_not_connected()
+			.unwrap()
+			.try_accept_incoming()
+			.is_ok());
 	}
 
 	#[test]
 	fn highest_not_connected_peer() {
-		let mut peers_state = PeersState::new(25, 25);
+		let mut peers_state = PeersState::new(iter::once(SetConfig {
+			in_peers: 25,
+			out_peers: 25,
+		}));
 		let id1 = PeerId::random();
 		let id2 = PeerId::random();
 
-		assert!(peers_state.highest_not_connected_peer().is_none());
+		assert!(peers_state.highest_not_connected_peer(0).is_none());
-		peers_state.peer(&id1).into_unknown().unwrap().discover().set_reputation(50);
+		peers_state
-		peers_state.peer(&id2).into_unknown().unwrap().discover().set_reputation(25);
+			.peer(0, &id1)
-		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+			.into_unknown()
-		peers_state.peer(&id2).into_not_connected().unwrap().set_reputation(75);
+			.unwrap()
-		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2.clone()));
+			.discover()
-		peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().unwrap();
+			.set_reputation(50);
-		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+		peers_state
-		peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(100);
+			.peer(0, &id2)
-		peers_state.peer(&id2).into_connected().unwrap().disconnect();
+			.into_unknown()
-		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
+			.unwrap()
-		peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(-100);
+			.discover()
-		assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2));
+			.set_reputation(25);
+		assert_eq!(
+			peers_state
+				.highest_not_connected_peer(0)
+				.map(|p| p.into_peer_id()),
+			Some(id1.clone())
+		);
+		peers_state
+			.peer(0, &id2)
+			.into_not_connected()
+			.unwrap()
+			.set_reputation(75);
+		assert_eq!(
+			peers_state
+				.highest_not_connected_peer(0)
+				.map(|p| p.into_peer_id()),
+			Some(id2.clone())
+		);
+		peers_state
+			.peer(0, &id2)
+			.into_not_connected()
+			.unwrap()
+			.try_accept_incoming()
+			.unwrap();
+		assert_eq!(
+			peers_state
+				.highest_not_connected_peer(0)
+				.map(|p| p.into_peer_id()),
+			Some(id1.clone())
+		);
+		peers_state
+			.peer(0, &id1)
+			.into_not_connected()
+			.unwrap()
+			.set_reputation(100);
+		peers_state
+			.peer(0, &id2)
+			.into_connected()
+			.unwrap()
+			.disconnect();
+		assert_eq!(
+			peers_state
+				.highest_not_connected_peer(0)
+				.map(|p| p.into_peer_id()),
+			Some(id1.clone())
+		);
+		peers_state
+			.peer(0, &id1)
+			.into_not_connected()
+			.unwrap()
+			.set_reputation(-100);
+		assert_eq!(
+			peers_state
+				.highest_not_connected_peer(0)
+				.map(|p| p.into_peer_id()),
+			Some(id2.clone())
+		);
 	}
 
 	#[test]
 	fn disconnect_no_slot_doesnt_panic() {
-		let mut peers_state = PeersState::new(1, 1);
+		let mut peers_state = PeersState::new(iter::once(SetConfig {
+			in_peers: 1,
+			out_peers: 1,
+		}));
 		let id = PeerId::random();
-		peers_state.add_no_slot_node(id.clone());
+		peers_state.add_no_slot_node(0, id.clone());
-		let peer = peers_state.peer(&id).into_unknown().unwrap().discover().try_outgoing().unwrap();
+		let peer = peers_state
+			.peer(0, &id)
+			.into_unknown()
+			.unwrap()
+			.discover()
+			.try_outgoing()
+			.unwrap();
 		peer.disconnect();
 	}
 }

substrate/client/peerset/tests/fuzz.rs

@@ -20,8 +20,8 @@ use futures::prelude::*;
 use libp2p::PeerId;
 use rand::distributions::{Distribution, Uniform, WeightedIndex};
 use rand::seq::IteratorRandom;
-use std::{collections::HashMap, collections::HashSet, iter, pin::Pin, task::Poll};
+use sc_peerset::{IncomingIndex, Message, Peerset, PeersetConfig, ReputationChange, SetConfig, SetId};
-use sc_peerset::{IncomingIndex, Message, PeersetConfig, Peerset, ReputationChange};
+use std::{collections::HashMap, collections::HashSet, pin::Pin, task::Poll};
 
 #[test]
 fn run() {
@@ -40,23 +40,30 @@ fn test_once() {
 	let mut reserved_nodes = HashSet::<PeerId>::new();
 
 	let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig {
-		bootnodes: (0 .. Uniform::new_inclusive(0, 4).sample(&mut rng)).map(|_| {
+		sets: vec![
-			let id = PeerId::random();
+			SetConfig {
-			known_nodes.insert(id.clone());
+				bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng))
-			id
+					.map(|_| {
-		}).collect(),
+						let id = PeerId::random();
-		priority_groups: {
+						known_nodes.insert(id.clone());
-			let nodes = (0 .. Uniform::new_inclusive(0, 2).sample(&mut rng)).map(|_| {
+						id
-				let id = PeerId::random();
+					})
-				known_nodes.insert(id.clone());
+					.collect(),
-				reserved_nodes.insert(id.clone());
+				reserved_nodes: {
-				id
+					(0..Uniform::new_inclusive(0, 2).sample(&mut rng))
-			}).collect();
+						.map(|_| {
-			vec![("foo".to_string(), nodes)]
+							let id = PeerId::random();
-		},
+							known_nodes.insert(id.clone());
-		reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0,
+							reserved_nodes.insert(id.clone());
-		in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+							id
-		out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+						})
+						.collect()
+				},
+				in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+				out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng),
+				reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0,
+			},
+		],
 	});
 
 	futures::executor::block_on(futures::future::poll_fn(move |cx| {
@@ -71,70 +78,101 @@ fn test_once() {
 
 		// Perform a certain number of actions while checking that the state is consistent. If we
 		// reach the end of the loop, the run has succeeded.
-		for _ in 0 .. 2500 {
+		for _ in 0..2500 {
 			// Each of these weights corresponds to an action that we may perform.
 			let action_weights = [150, 90, 90, 30, 30, 1, 1, 4, 4];
-			match WeightedIndex::new(&action_weights).unwrap().sample(&mut rng) {
+			match WeightedIndex::new(&action_weights)
+				.unwrap()
+				.sample(&mut rng)
+			{
 				// If we generate 0, poll the peerset.
 				0 => match Stream::poll_next(Pin::new(&mut peerset), cx) {
-					Poll::Ready(Some(Message::Connect(id))) => {
+					Poll::Ready(Some(Message::Connect { peer_id, .. })) => {
-						if let Some(id) = incoming_nodes.iter().find(|(_, v)| **v == id).map(|(&id, _)| id) {
+						if let Some(id) = incoming_nodes
+							.iter()
+							.find(|(_, v)| **v == peer_id)
+							.map(|(&id, _)| id)
+						{
 							incoming_nodes.remove(&id);
 						}
-						assert!(connected_nodes.insert(id));
+						assert!(connected_nodes.insert(peer_id));
+					}
+					Poll::Ready(Some(Message::Drop { peer_id, .. })) => {
+						connected_nodes.remove(&peer_id);
+					}
+					Poll::Ready(Some(Message::Accept(n))) => {
+						assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap()))
+					}
+					Poll::Ready(Some(Message::Reject(n))) => {
+						assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap()))
 					}
-					Poll::Ready(Some(Message::Drop(id))) => { connected_nodes.remove(&id); }
-					Poll::Ready(Some(Message::Accept(n))) =>
-						assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap())),
-					Poll::Ready(Some(Message::Reject(n))) =>
-						assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap())),
 					Poll::Ready(None) => panic!(),
 					Poll::Pending => {}
-				}
+				},
 
 				// If we generate 1, discover a new node.
 				1 => {
 					let new_id = PeerId::random();
 					known_nodes.insert(new_id.clone());
-					peerset.discovered(iter::once(new_id));
+					peerset.add_to_peers_set(SetId::from(0), new_id);
 				}
 
 				// If we generate 2, adjust a random reputation.
-				2 => if let Some(id) = known_nodes.iter().choose(&mut rng) {
+				2 => {
-					let val = Uniform::new_inclusive(i32::min_value(), i32::max_value()).sample(&mut rng);
+					if let Some(id) = known_nodes.iter().choose(&mut rng) {
-					peerset_handle.report_peer(id.clone(), ReputationChange::new(val, ""));
+						let val = Uniform::new_inclusive(i32::min_value(), i32::max_value())
+							.sample(&mut rng);
+						peerset_handle.report_peer(id.clone(), ReputationChange::new(val, ""));
+					}
 				}
 
 				// If we generate 3, disconnect from a random node.
-				3 => if let Some(id) = connected_nodes.iter().choose(&mut rng).cloned() {
+				3 => {
-					connected_nodes.remove(&id);
+					if let Some(id) = connected_nodes.iter().choose(&mut rng).cloned() {
-					peerset.dropped(id);
+						connected_nodes.remove(&id);
+						peerset.dropped(SetId::from(0), id);
+					}
 				}
 
 				// If we generate 4, connect to a random node.
-				4 => if let Some(id) = known_nodes.iter()
+				4 => {
-					.filter(|n| incoming_nodes.values().all(|m| m != *n) && !connected_nodes.contains(*n))
+					if let Some(id) = known_nodes
-					.choose(&mut rng) {
+						.iter()
-					peerset.incoming(id.clone(), next_incoming_id);
+						.filter(|n| {
-					incoming_nodes.insert(next_incoming_id, id.clone());
+							incoming_nodes.values().all(|m| m != *n)
-					next_incoming_id.0 += 1;
+								&& !connected_nodes.contains(*n)
+						})
+						.choose(&mut rng)
+					{
+						peerset.incoming(SetId::from(0), id.clone(), next_incoming_id.clone());
+						incoming_nodes.insert(next_incoming_id.clone(), id.clone());
+						next_incoming_id.0 += 1;
+					}
 				}
 
 				// 5 and 6 are the reserved-only mode.
-				5 => peerset_handle.set_reserved_only(true),
+				5 => peerset_handle.set_reserved_only(SetId::from(0), true),
-				6 => peerset_handle.set_reserved_only(false),
+				6 => peerset_handle.set_reserved_only(SetId::from(0), false),
 
 				// 7 and 8 are about switching a random node in or out of reserved mode.
-				7 => if let Some(id) = known_nodes.iter().filter(|n| !reserved_nodes.contains(*n)).choose(&mut rng) {
+				7 => {
-					peerset_handle.add_reserved_peer(id.clone());
+					if let Some(id) = known_nodes
-					reserved_nodes.insert(id.clone());
+						.iter()
+						.filter(|n| !reserved_nodes.contains(*n))
+						.choose(&mut rng)
+					{
+						peerset_handle.add_reserved_peer(SetId::from(0), id.clone());
+						reserved_nodes.insert(id.clone());
+					}
 				}
-				8 => if let Some(id) = reserved_nodes.iter().choose(&mut rng).cloned() {
+				8 => {
-					reserved_nodes.remove(&id);
+					if let Some(id) = reserved_nodes.iter().choose(&mut rng).cloned() {
-					peerset_handle.remove_reserved_peer(id);
+						reserved_nodes.remove(&id);
+						peerset_handle.remove_reserved_peer(SetId::from(0), id);
+					}
 				}
 
-				_ => unreachable!()
+				_ => unreachable!(),
 			}
 		}