pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-14 06:21:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework priority groups, take 2 (#7700)

Browse Source 
* Rework priority groups

* Broken tests fix

* Fix warning causing CI to fail

* [Hack] Try restore backwards-compatibility

* Fix peerset bug

* Doc fixes and clean up

* Error on state mismatch

* Try debug CI

* CI debugging

* [CI debug] Can I please see this line

* Revert "[CI debug] Can I please see this line"

This reverts commit 4b7cf7c1511f579cd818b21d46bd11642dfac5cb.

* Revert "CI debugging"

This reverts commit 9011f1f564b860386dc7dd6ffa9fc34ea7107623.

* Fix error! which isn't actually an error

* Fix Ok() returned when actually Err()

* Tweaks and fixes

* Fix build

* Peerset bugfix

* [Debug] Try outbound GrandPa slots

* Another bugfix

* Revert "[Debug] Try outbound GrandPa slots"

This reverts commit d175b9208c088faad77d9f0ce36ff6f48bd92dd3.

* [Debug] Try outbound GrandPa slots

* Apply suggestions from code review

Co-authored-by: Max Inden <mail@max-inden.de>

* Use consts for hardcoded peersets

* Revert "Try debug CI"

This reverts commit 62c4ad5e79c03d561c714a008022ecac463a597e.

* Renames

* Line widths

* Add doc

Co-authored-by: Max Inden <mail@max-inden.de>
This commit is contained in:
Pierre Krieger
2021-01-07 14:52:39 +01:00
committed by GitHub
parent 94bb119ef9
commit 779c4f8616
30 changed files with 2742 additions and 2293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/peerset/src/lib.rs
+380 -350
View File
@@ -18,14 +18,27 @@
//! Peer Set Manager (PSM). Contains the strategy for choosing which nodes the network should be
//! connected to.
//!
//! The PSM handles *sets* of nodes. A set of nodes is defined as the nodes that are believed to
//! support a certain capability, such as handling blocks and transactions of a specific chain,
//! or collating a certain parachain.
//!
//! For each node in each set, the peerset holds a flag specifying whether the node is
//! connected to us or not.
//!
//! This connected/disconnected status is specific to the node and set combination, and it is for
//! example possible for a node to be connected through a specific set but not another.
//!
//! In addition, for each, set, the peerset also holds a list of reserved nodes towards which it
//! will at all time try to maintain a connection with.
mod peersstate;
use std::{collections::{HashSet, HashMap}, collections::VecDeque};
use std::{collections::HashSet, collections::VecDeque};
use futures::prelude::*;
use log::{debug, error, trace};
use serde_json::json;
use std::{pin::Pin, task::{Context, Poll}, time::Duration};
use std::{collections::HashMap, pin::Pin, task::{Context, Poll}, time::Duration};
use wasm_timer::Instant;
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedSender, TracingUnboundedReceiver};
@@ -35,22 +48,46 @@ pub use libp2p::PeerId;
const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100);
/// Reputation change for a node when we get disconnected from it.
const DISCONNECT_REPUTATION_CHANGE: i32 = -256;
/// Reserved peers group ID
const RESERVED_NODES: &str = "reserved";
/// Amount of time between the moment we disconnect from a node and the moment we remove it from
/// the list.
const FORGET_AFTER: Duration = Duration::from_secs(3600);
#[derive(Debug)]
enum Action {
AddReservedPeer(PeerId),
RemoveReservedPeer(PeerId),
SetReservedPeers(HashSet<PeerId>),
SetReservedOnly(bool),
AddReservedPeer(SetId, PeerId),
RemoveReservedPeer(SetId, PeerId),
SetReservedPeers(SetId, HashSet<PeerId>),
SetReservedOnly(SetId, bool),
ReportPeer(PeerId, ReputationChange),
SetPriorityGroup(String, HashSet<PeerId>),
AddToPriorityGroup(String, PeerId),
RemoveFromPriorityGroup(String, PeerId),
AddToPeersSet(SetId, PeerId),
RemoveFromPeersSet(SetId, PeerId),
}
/// Identifier of a set in the peerset.
///
/// Can be constructed using the `From<usize>` trait implementation based on the index of the set
/// within [`PeersetConfig::sets`]. For example, the first element of [`PeersetConfig::sets`] is
/// later referred to with `SetId::from(0)`. It is intended that the code responsible for building
/// the [`PeersetConfig`] is also responsible for constructing the [`SetId`]s.
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SetId(usize);
impl SetId {
pub const fn from(id: usize) -> Self {
SetId(id)
}
}
impl From<usize> for SetId {
fn from(id: usize) -> Self {
SetId(id)
}
}
impl From<SetId> for usize {
fn from(id: SetId) -> Self {
id.0
}
}
/// Description of a reputation adjustment for a node.
@@ -88,25 +125,26 @@ impl PeersetHandle {
///
/// > **Note**: Keep in mind that the networking has to know an address for this node,
/// > otherwise it will not be able to connect to it.
pub fn add_reserved_peer(&self, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::AddReservedPeer(peer_id));
pub fn add_reserved_peer(&self, set_id: SetId, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::AddReservedPeer(set_id, peer_id));
}
/// Remove a previously-added reserved peer.
///
/// Has no effect if the node was not a reserved peer.
pub fn remove_reserved_peer(&self, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(peer_id));
pub fn remove_reserved_peer(&self, set_id: SetId, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(set_id, peer_id));
}
/// Sets whether or not the peerset only has connections .
pub fn set_reserved_only(&self, reserved: bool) {
let _ = self.tx.unbounded_send(Action::SetReservedOnly(reserved));
/// Sets whether or not the peerset only has connections with nodes marked as reserved for
/// the given set.
pub fn set_reserved_only(&self, set_id: SetId, reserved: bool) {
let _ = self.tx.unbounded_send(Action::SetReservedOnly(set_id, reserved));
}
/// Set reserved peers to the new set.
pub fn set_reserved_peers(&self, peer_ids: HashSet<PeerId>) {
let _ = self.tx.unbounded_send(Action::SetReservedPeers(peer_ids));
pub fn set_reserved_peers(&self, set_id: SetId, peer_ids: HashSet<PeerId>) {
let _ = self.tx.unbounded_send(Action::SetReservedPeers(set_id, peer_ids));
}
/// Reports an adjustment to the reputation of the given peer.
@@ -114,19 +152,14 @@ impl PeersetHandle {
let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff));
}
/// Modify a priority group.
pub fn set_priority_group(&self, group_id: String, peers: HashSet<PeerId>) {
let _ = self.tx.unbounded_send(Action::SetPriorityGroup(group_id, peers));
/// Add a peer to a set.
pub fn add_to_peers_set(&self, set_id: SetId, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::AddToPeersSet(set_id, peer_id));
}
/// Add a peer to a priority group.
pub fn add_to_priority_group(&self, group_id: String, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::AddToPriorityGroup(group_id, peer_id));
}
/// Remove a peer from a priority group.
pub fn remove_from_priority_group(&self, group_id: String, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::RemoveFromPriorityGroup(group_id, peer_id));
/// Remove a peer from a set.
pub fn remove_from_peers_set(&self, set_id: SetId, peer_id: PeerId) {
let _ = self.tx.unbounded_send(Action::RemoveFromPeersSet(set_id, peer_id));
}
}
@@ -135,10 +168,18 @@ impl PeersetHandle {
pub enum Message {
/// Request to open a connection to the given peer. From the point of view of the PSM, we are
/// immediately connected.
Connect(PeerId),
Connect {
set_id: SetId,
/// Peer to connect to.
peer_id: PeerId,
},
/// Drop the connection to the given peer, or cancel the connection attempt after a `Connect`.
Drop(PeerId),
Drop {
set_id: SetId,
/// Peer to disconnect from.
peer_id: PeerId,
},
/// Equivalent to `Connect` for the peer corresponding to this incoming index.
Accept(IncomingIndex),
@@ -160,26 +201,33 @@ impl From<u64> for IncomingIndex {
/// Configuration to pass when creating the peer set manager.
#[derive(Debug)]
pub struct PeersetConfig {
/// List of sets of nodes the peerset manages.
pub sets: Vec<SetConfig>,
}
/// Configuration for a single set of nodes.
#[derive(Debug)]
pub struct SetConfig {
/// Maximum number of ingoing links to peers.
pub in_peers: u32,
/// Maximum number of outgoing links to peers.
pub out_peers: u32,
/// List of bootstrap nodes to initialize the peer with.
/// List of bootstrap nodes to initialize the set with.
///
/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
/// > otherwise it will not be able to connect to them.
pub bootnodes: Vec<PeerId>,
/// If true, we only accept nodes in [`PeersetConfig::priority_groups`].
pub reserved_only: bool,
/// Lists of nodes we should always be connected to.
///
/// > **Note**: Keep in mind that the networking has to know an address for these nodes,
/// > otherwise it will not be able to connect to them.
pub priority_groups: Vec<(String, HashSet<PeerId>)>,
/// > otherwise it will not be able to connect to them.
pub reserved_nodes: HashSet<PeerId>,
/// If true, we only accept nodes in [`SetConfig::reserved_nodes`].
pub reserved_only: bool,
}
/// Side of the peer set manager owned by the network. In other words, the "receiving" side.
@@ -190,11 +238,10 @@ pub struct PeersetConfig {
pub struct Peerset {
/// Underlying data structure for the nodes's states.
data: peersstate::PeersState,
/// If true, we only accept reserved nodes.
reserved_only: bool,
/// Lists of nodes that don't occupy slots and that we should try to always be connected to.
/// Is kept in sync with the list of reserved nodes in [`Peerset::data`].
priority_groups: HashMap<String, HashSet<PeerId>>,
/// For each set, lists of nodes that don't occupy slots and that we should try to always be
/// connected to, and whether only reserved nodes are accepted. Is kept in sync with the list
/// of non-slot-occupying nodes in [`Peerset::data`].
reserved_nodes: Vec<(HashSet<PeerId>, bool)>,
/// Receiver for messages from the `PeersetHandle` and from `tx`.
rx: TracingUnboundedReceiver<Action>,
/// Sending side of `rx`.
@@ -216,28 +263,36 @@ impl Peerset {
tx: tx.clone(),
};
let now = Instant::now();
let mut peerset = {
let now = Instant::now();
let mut peerset = Peerset {
data: peersstate::PeersState::new(config.in_peers, config.out_peers),
tx,
rx,
reserved_only: config.reserved_only,
priority_groups: config.priority_groups.clone().into_iter().collect(),
message_queue: VecDeque::new(),
created: now,
latest_time_update: now,
Peerset {
data: peersstate::PeersState::new(config.sets.iter().map(|set| peersstate::SetConfig {
in_peers: set.in_peers,
out_peers: set.out_peers,
})),
tx,
rx,
reserved_nodes: config.sets.iter().map(|set| {
(set.reserved_nodes.clone(), set.reserved_only)
}).collect(),
message_queue: VecDeque::new(),
created: now,
latest_time_update: now,
}
};
for node in config.priority_groups.into_iter().flat_map(|(_, l)| l) {
peerset.data.add_no_slot_node(node);
}
for (set, set_config) in config.sets.into_iter().enumerate() {
for node in set_config.reserved_nodes {
peerset.data.add_no_slot_node(set, node);
}
for peer_id in config.bootnodes {
if let peersstate::Peer::Unknown(entry) = peerset.data.peer(&peer_id) {
entry.discover();
} else {
debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id);
for peer_id in set_config.bootnodes {
if let peersstate::Peer::Unknown(entry) = peerset.data.peer(set, &peer_id) {
entry.discover();
} else {
debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id);
}
}
}
@@ -245,96 +300,109 @@ impl Peerset {
(peerset, handle)
}
fn on_add_reserved_peer(&mut self, peer_id: PeerId) {
self.on_add_to_priority_group(RESERVED_NODES, peer_id);
fn on_add_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) {
let newly_inserted = self.reserved_nodes[set_id.0].0.insert(peer_id.clone());
if !newly_inserted {
return;
}
self.data.add_no_slot_node(set_id.0, peer_id);
self.alloc_slots();
}
fn on_remove_reserved_peer(&mut self, peer_id: PeerId) {
self.on_remove_from_priority_group(RESERVED_NODES, peer_id);
}
fn on_remove_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) {
if !self.reserved_nodes[set_id.0].0.remove(&peer_id) {
return;
}
fn on_set_reserved_peers(&mut self, peer_ids: HashSet<PeerId>) {
self.on_set_priority_group(RESERVED_NODES, peer_ids);
}
self.data.remove_no_slot_node(set_id.0, &peer_id);
fn on_set_reserved_only(&mut self, reserved_only: bool) {
self.reserved_only = reserved_only;
// Nothing more to do if not in reserved-only mode.
if !self.reserved_nodes[set_id.0].1 {
return;
}
if self.reserved_only {
// Disconnect all the nodes that aren't reserved.
for peer_id in self.data.connected_peers().cloned().collect::<Vec<_>>().into_iter() {
if self.priority_groups.get(RESERVED_NODES).map_or(false, |g| g.contains(&peer_id)) {
continue;
}
let peer = self.data.peer(&peer_id).into_connected()
.expect("We are enumerating connected peers, therefore the peer is connected; qed");
peer.disconnect();
self.message_queue.push_back(Message::Drop(peer_id));
}
} else {
self.alloc_slots();
// If, however, the peerset is in reserved-only mode, then the removed node needs to be
// disconnected.
if let peersstate::Peer::Connected(peer) = self.data.peer(set_id.0, &peer_id) {
peer.disconnect();
self.message_queue.push_back(Message::Drop {
set_id,
peer_id,
});
}
}
fn on_set_priority_group(&mut self, group_id: &str, peers: HashSet<PeerId>) {
fn on_set_reserved_peers(&mut self, set_id: SetId, peer_ids: HashSet<PeerId>) {
// Determine the difference between the current group and the new list.
let (to_insert, to_remove) = {
let current_group = self.priority_groups.entry(group_id.to_owned()).or_default();
let to_insert = peers.difference(current_group)
let to_insert = peer_ids.difference(&self.reserved_nodes[set_id.0].0)
.cloned().collect::<Vec<_>>();
let to_remove = current_group.difference(&peers)
let to_remove = self.reserved_nodes[set_id.0].0.difference(&peer_ids)
.cloned().collect::<Vec<_>>();
(to_insert, to_remove)
};
// Enumerate elements in `peers` not in `current_group`.
for peer_id in &to_insert {
// We don't call `on_add_to_priority_group` here in order to avoid calling
// `alloc_slots` all the time.
self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone());
self.data.add_no_slot_node(peer_id.clone());
for node in to_insert {
self.on_add_reserved_peer(set_id, node);
}
// Enumerate elements in `current_group` not in `peers`.
for peer in to_remove {
self.on_remove_from_priority_group(group_id, peer);
for node in to_remove {
self.on_remove_reserved_peer(set_id, node);
}
}
if !to_insert.is_empty() {
fn on_set_reserved_only(&mut self, set_id: SetId, reserved_only: bool) {
self.reserved_nodes[set_id.0].1 = reserved_only;
if reserved_only {
// Disconnect all the nodes that aren't reserved.
for peer_id in self.data.connected_peers(set_id.0).cloned().collect::<Vec<_>>().into_iter() {
if self.reserved_nodes[set_id.0].0.contains(&peer_id) {
continue;
}
let peer = self.data.peer(set_id.0, &peer_id).into_connected()
.expect("We are enumerating connected peers, therefore the peer is connected; qed");
peer.disconnect();
self.message_queue.push_back(Message::Drop {
set_id,
peer_id
});
}
} else {
self.alloc_slots();
}
}
fn on_add_to_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
self.priority_groups.entry(group_id.to_owned()).or_default().insert(peer_id.clone());
self.data.add_no_slot_node(peer_id);
self.alloc_slots();
/// Adds a node to the given set. The peerset will, if possible and not already the case,
/// try to connect to it.
///
/// > **Note**: This has the same effect as [`PeersetHandle::add_to_peers_set`].
pub fn add_to_peers_set(&mut self, set_id: SetId, peer_id: PeerId) {
if let peersstate::Peer::Unknown(entry) = self.data.peer(set_id.0, &peer_id) {
entry.discover();
self.alloc_slots();
}
}
fn on_remove_from_priority_group(&mut self, group_id: &str, peer_id: PeerId) {
if let Some(priority_group) = self.priority_groups.get_mut(group_id) {
if !priority_group.remove(&peer_id) {
// `PeerId` wasn't in the group in the first place.
return;
}
} else {
// Group doesn't exist, so the `PeerId` can't be in it.
fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) {
// Don't do anything if node is reserved.
if self.reserved_nodes[set_id.0].0.contains(&peer_id) {
return;
}
// If that `PeerId` isn't in any other group, then it is no longer no-slot-occupying.
if !self.priority_groups.values().any(|l| l.contains(&peer_id)) {
self.data.remove_no_slot_node(&peer_id);
}
// Disconnect the peer if necessary.
if group_id != RESERVED_NODES && self.reserved_only {
if let peersstate::Peer::Connected(peer) = self.data.peer(&peer_id) {
peer.disconnect();
self.message_queue.push_back(Message::Drop(peer_id));
match self.data.peer(set_id.0, &peer_id) {
peersstate::Peer::Connected(peer) => {
self.message_queue.push_back(Message::Drop {
set_id,
peer_id: peer.peer_id().clone(),
});
peer.disconnect().forget_peer();
}
peersstate::Peer::NotConnected(peer) => { peer.forget_peer(); }
peersstate::Peer::Unknown(_) => {}
}
}
@@ -342,33 +410,29 @@ impl Peerset {
// We want reputations to be up-to-date before adjusting them.
self.update_time();
match self.data.peer(&peer_id) {
peersstate::Peer::Connected(mut peer) => {
peer.add_reputation(change.value);
if peer.reputation() < BANNED_THRESHOLD {
debug!(target: "peerset", "Report {}: {:+} to {}. Reason: {}, Disconnecting",
peer_id, change.value, peer.reputation(), change.reason
);
peer.disconnect();
self.message_queue.push_back(Message::Drop(peer_id));
} else {
trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
peer_id, change.value, peer.reputation(), change.reason
);
}
},
peersstate::Peer::NotConnected(mut peer) => {
trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
peer_id, change.value, peer.reputation(), change.reason
);
peer.add_reputation(change.value)
},
peersstate::Peer::Unknown(peer) => {
trace!(target: "peerset", "Discover {}: {:+}. Reason: {}",
peer_id, change.value, change.reason
);
peer.discover().add_reputation(change.value)
},
let mut reputation = self.data.peer_reputation(peer_id.clone());
reputation.add_reputation(change.value);
if reputation.reputation() >= BANNED_THRESHOLD {
trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}",
peer_id, change.value, reputation.reputation(), change.reason
);
return;
}
debug!(target: "peerset", "Report {}: {:+} to {}. Reason: {}, Disconnecting",
peer_id, change.value, reputation.reputation(), change.reason
);
drop(reputation);
for set_index in 0..self.data.num_sets() {
if let peersstate::Peer::Connected(peer) = self.data.peer(set_index, &peer_id) {
let peer = peer.disconnect();
self.message_queue.push_back(Message::Drop {
set_id: SetId(set_index),
peer_id: peer.into_peer_id(),
});
}
}
}
@@ -403,27 +467,35 @@ impl Peerset {
}
reput.saturating_sub(diff)
}
match self.data.peer(&peer_id) {
peersstate::Peer::Connected(mut peer) => {
let before = peer.reputation();
let after = reput_tick(before);
trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
peer.set_reputation(after)
}
peersstate::Peer::NotConnected(mut peer) => {
if peer.reputation() == 0 &&
peer.last_connected_or_discovered() + FORGET_AFTER < now
{
peer.forget_peer();
} else {
let before = peer.reputation();
let after = reput_tick(before);
trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
peer.set_reputation(after)
let mut peer_reputation = self.data.peer_reputation(peer_id.clone());
let before = peer_reputation.reputation();
let after = reput_tick(before);
trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after);
peer_reputation.set_reputation(after);
if after != 0 {
continue;
}
drop(peer_reputation);
// If the peer reaches a reputation of 0, and there is no connection to it,
// forget it.
for set_index in 0..self.data.num_sets() {
match self.data.peer(set_index, &peer_id) {
peersstate::Peer::Connected(_) => {}
peersstate::Peer::NotConnected(peer) => {
if peer.last_connected_or_discovered() + FORGET_AFTER < now {
peer.forget_peer();
}
}
peersstate::Peer::Unknown(_) => {
// Happens if this peer does not belong to this set.
}
}
peersstate::Peer::Unknown(_) => unreachable!("We iterate over known peers; qed")
};
}
}
}
}
@@ -433,89 +505,54 @@ impl Peerset {
self.update_time();
// Try to connect to all the reserved nodes that we are not connected to.
loop {
let next = {
let data = &mut self.data;
self.priority_groups
.get(RESERVED_NODES)
.into_iter()
.flatten()
.find(move |n| {
data.peer(n).into_connected().is_none()
})
.cloned()
};
for set_index in 0..self.data.num_sets() {
for reserved_node in &self.reserved_nodes[set_index].0 {
let entry = match self.data.peer(set_index, reserved_node) {
peersstate::Peer::Unknown(n) => n.discover(),
peersstate::Peer::NotConnected(n) => n,
peersstate::Peer::Connected(_) => continue,
};
let next = match next {
Some(n) => n,
None => break,
};
match entry.try_outgoing() {
Ok(conn) => self.message_queue.push_back(Message::Connect {
set_id: SetId(set_index),
peer_id: conn.into_peer_id()
}),
Err(_) => {
// An error is returned only if no slot is available. Reserved nodes are
// marked in the state machine with a flag saying "doesn't occupy a slot",
// and as such this should never happen.
debug_assert!(false);
log::error!(
target: "peerset",
"Not enough slots to connect to reserved node"
);
}
}
}
}
let next = match self.data.peer(&next) {
peersstate::Peer::Unknown(n) => n.discover(),
peersstate::Peer::NotConnected(n) => n,
peersstate::Peer::Connected(_) => {
debug_assert!(false, "State inconsistency: not connected state");
// Now, we try to connect to other nodes.
for set_index in 0..self.data.num_sets() {
// Nothing more to do if we're in reserved mode.
if self.reserved_nodes[set_index].1 {
continue;
}
// Try to grab the next node to attempt to connect to.
while let Some(next) = self.data.highest_not_connected_peer(set_index) {
// Don't connect to nodes with an abysmal reputation.
if next.reputation() < BANNED_THRESHOLD {
break;
}
};
match next.try_outgoing() {
Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
Err(_) => break, // No more slots available.
}
}
// Nothing more to do if we're in reserved mode.
if self.reserved_only {
return;
}
// Try to connect to all the nodes in priority groups and that we are not connected to.
loop {
let next = {
let data = &mut self.data;
self.priority_groups
.values()
.flatten()
.find(move |n| {
data.peer(n).into_connected().is_none()
})
.cloned()
};
let next = match next {
Some(n) => n,
None => break,
};
let next = match self.data.peer(&next) {
peersstate::Peer::Unknown(n) => n.discover(),
peersstate::Peer::NotConnected(n) => n,
peersstate::Peer::Connected(_) => {
debug_assert!(false, "State inconsistency: not connected state");
break;
match next.try_outgoing() {
Ok(conn) => self.message_queue.push_back(Message::Connect {
set_id: SetId(set_index),
peer_id: conn.into_peer_id()
}),
Err(_) => break, // No more slots available.
}
};
match next.try_outgoing() {
Ok(conn) => self.message_queue.push_back(Message::Connect(conn.into_peer_id())),
Err(_) => break, // No more slots available.
}
}
// Now, we try to connect to non-priority nodes.
while let Some(next) = self.data.highest_not_connected_peer() {
// Don't connect to nodes with an abysmal reputation.
if next.reputation() < BANNED_THRESHOLD {
break;
}
match next.try_outgoing() {
Ok(conn) => self
.message_queue
.push_back(Message::Connect(conn.into_peer_id())),
Err(_) => break, // No more slots available.
}
}
}
@@ -530,16 +567,19 @@ impl Peerset {
// Implementation note: because of concurrency issues, it is possible that we push a `Connect`
// message to the output channel with a `PeerId`, and that `incoming` gets called with the same
// `PeerId` before that message has been read by the user. In this situation we must not answer.
pub fn incoming(&mut self, peer_id: PeerId, index: IncomingIndex) {
pub fn incoming(&mut self, set_id: SetId, peer_id: PeerId, index: IncomingIndex) {
trace!(target: "peerset", "Incoming {:?}", peer_id);
self.update_time();
if self.reserved_only && !self.priority_groups.get(RESERVED_NODES).map_or(false, |n| n.contains(&peer_id)) {
self.message_queue.push_back(Message::Reject(index));
return;
if self.reserved_nodes[set_id.0].1 {
if !self.reserved_nodes[set_id.0].0.contains(&peer_id) {
self.message_queue.push_back(Message::Reject(index));
return;
}
}
let not_connected = match self.data.peer(&peer_id) {
let not_connected = match self.data.peer(set_id.0, &peer_id) {
// If we're already connected, don't answer, as the docs mention.
peersstate::Peer::Connected(_) => return,
peersstate::Peer::NotConnected(mut entry) => {
@@ -564,11 +604,11 @@ impl Peerset {
///
/// Must only be called after the PSM has either generated a `Connect` message with this
/// `PeerId`, or accepted an incoming connection with this `PeerId`.
pub fn dropped(&mut self, peer_id: PeerId) {
pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId) {
// We want reputations to be up-to-date before adjusting them.
self.update_time();
match self.data.peer(&peer_id) {
match self.data.peer(set_id.0, &peer_id) {
peersstate::Peer::Connected(mut entry) => {
// Decrease the node's reputation so that we don't try it again and again and again.
entry.add_reputation(DISCONNECT_REPUTATION_CHANGE);
@@ -583,25 +623,6 @@ impl Peerset {
self.alloc_slots();
}
/// Adds discovered peer ids to the PSM.
///
/// > **Note**: There is no equivalent "expired" message, meaning that it is the responsibility
/// > of the PSM to remove `PeerId`s that fail to dial too often.
pub fn discovered<I: IntoIterator<Item = PeerId>>(&mut self, peer_ids: I) {
let mut discovered_any = false;
for peer_id in peer_ids {
if let peersstate::Peer::Unknown(entry) = self.data.peer(&peer_id) {
entry.discover();
discovered_any = true;
}
}
if discovered_any {
self.alloc_slots();
}
}
/// Reports an adjustment to the reputation of the given peer.
pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) {
// We don't immediately perform the adjustments in order to have state consistency. We
@@ -615,23 +636,29 @@ impl Peerset {
self.update_time();
json!({
"nodes": self.data.peers().cloned().collect::<Vec<_>>().into_iter().map(|peer_id| {
let state = match self.data.peer(&peer_id) {
peersstate::Peer::Connected(entry) => json!({
"connected": true,
"reputation": entry.reputation()
}),
peersstate::Peer::NotConnected(entry) => json!({
"connected": false,
"reputation": entry.reputation()
}),
peersstate::Peer::Unknown(_) =>
unreachable!("We iterate over the known peers; QED")
};
"sets": (0..self.data.num_sets()).map(|set_index| {
json!({
"nodes": self.data.peers().cloned().collect::<Vec<_>>().into_iter().filter_map(|peer_id| {
let state = match self.data.peer(set_index, &peer_id) {
peersstate::Peer::Connected(entry) => json!({
"connected": true,
"reputation": entry.reputation()
}),
peersstate::Peer::NotConnected(entry) => json!({
"connected": false,
"reputation": entry.reputation()
}),
peersstate::Peer::Unknown(_) => return None,
};
(peer_id.to_base58(), state)
}).collect::<HashMap<_, _>>(),
"reserved_only": self.reserved_only,
Some((peer_id.to_base58(), state))
}).collect::<HashMap<_, _>>(),
"reserved_nodes": self.reserved_nodes[set_index].0.iter().map(|peer_id| {
peer_id.to_base58()
}).collect::<HashSet<_>>(),
"reserved_only": self.reserved_nodes[set_index].1,
})
}).collect::<Vec<_>>(),
"message_queue": self.message_queue.len(),
})
}
@@ -640,11 +667,6 @@ impl Peerset {
pub fn num_discovered_peers(&self) -> usize {
self.data.peers().len()
}
/// Returns the content of a priority group.
pub fn priority_group(&self, group_id: &str) -> Option<impl ExactSizeIterator<Item = &PeerId>> {
self.priority_groups.get(group_id).map(|l| l.iter())
}
}
impl Stream for Peerset {
@@ -663,22 +685,20 @@ impl Stream for Peerset {
};
match action {
Action::AddReservedPeer(peer_id) =>
self.on_add_reserved_peer(peer_id),
Action::RemoveReservedPeer(peer_id) =>
self.on_remove_reserved_peer(peer_id),
Action::SetReservedPeers(peer_ids) =>
self.on_set_reserved_peers(peer_ids),
Action::SetReservedOnly(reserved) =>
self.on_set_reserved_only(reserved),
Action::AddReservedPeer(set_id, peer_id) =>
self.on_add_reserved_peer(set_id, peer_id),
Action::RemoveReservedPeer(set_id, peer_id) =>
self.on_remove_reserved_peer(set_id, peer_id),
Action::SetReservedPeers(set_id, peer_ids) =>
self.on_set_reserved_peers(set_id, peer_ids),
Action::SetReservedOnly(set_id, reserved) =>
self.on_set_reserved_only(set_id, reserved),
Action::ReportPeer(peer_id, score_diff) =>
self.on_report_peer(peer_id, score_diff),
Action::SetPriorityGroup(group_id, peers) =>
self.on_set_priority_group(&group_id, peers),
Action::AddToPriorityGroup(group_id, peer_id) =>
self.on_add_to_priority_group(&group_id, peer_id),
Action::RemoveFromPriorityGroup(group_id, peer_id) =>
self.on_remove_from_priority_group(&group_id, peer_id),
Action::AddToPeersSet(sets_name, peer_id) =>
self.add_to_peers_set(sets_name, peer_id),
Action::RemoveFromPeersSet(sets_name, peer_id) =>
self.on_remove_from_peers_set(sets_name, peer_id),
}
}
}
@@ -688,7 +708,7 @@ impl Stream for Peerset {
mod tests {
use libp2p::PeerId;
use futures::prelude::*;
use super::{PeersetConfig, Peerset, Message, IncomingIndex, ReputationChange, BANNED_THRESHOLD};
use super::{PeersetConfig, Peerset, Message, IncomingIndex, ReputationChange, SetConfig, SetId, BANNED_THRESHOLD};
use std::{pin::Pin, task::Poll, thread, time::Duration};
fn assert_messages(mut peerset: Peerset, messages: Vec<Message>) -> Peerset {
@@ -712,20 +732,22 @@ mod tests {
let reserved_peer = PeerId::random();
let reserved_peer2 = PeerId::random();
let config = PeersetConfig {
in_peers: 0,
out_peers: 2,
bootnodes: vec![bootnode],
reserved_only: true,
priority_groups: Vec::new(),
sets: vec![SetConfig {
in_peers: 0,
out_peers: 2,
bootnodes: vec![bootnode],
reserved_nodes: Default::default(),
reserved_only: true,
}],
};
let (peerset, handle) = Peerset::from_config(config);
handle.add_reserved_peer(reserved_peer.clone());
handle.add_reserved_peer(reserved_peer2.clone());
handle.add_reserved_peer(SetId::from(0), reserved_peer.clone());
handle.add_reserved_peer(SetId::from(0), reserved_peer2.clone());
assert_messages(peerset, vec![
Message::Connect(reserved_peer),
Message::Connect(reserved_peer2)
Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer },
Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer2 }
]);
}
@@ -740,21 +762,23 @@ mod tests {
let ii3 = IncomingIndex(3);
let ii4 = IncomingIndex(3);
let config = PeersetConfig {
in_peers: 2,
out_peers: 1,
bootnodes: vec![bootnode.clone()],
reserved_only: false,
priority_groups: Vec::new(),
sets: vec![SetConfig {
in_peers: 2,
out_peers: 1,
bootnodes: vec![bootnode.clone()],
reserved_nodes: Default::default(),
reserved_only: false,
}],
};
let (mut peerset, _handle) = Peerset::from_config(config);
peerset.incoming(incoming.clone(), ii);
peerset.incoming(incoming, ii4);
peerset.incoming(incoming2, ii2);
peerset.incoming(incoming3, ii3);
peerset.incoming(SetId::from(0), incoming.clone(), ii);
peerset.incoming(SetId::from(0), incoming.clone(), ii4);
peerset.incoming(SetId::from(0), incoming2.clone(), ii2);
peerset.incoming(SetId::from(0), incoming3.clone(), ii3);
assert_messages(peerset, vec![
Message::Connect(bootnode),
Message::Connect { set_id: SetId::from(0), peer_id: bootnode.clone() },
Message::Accept(ii),
Message::Accept(ii2),
Message::Reject(ii3),
@@ -766,15 +790,17 @@ mod tests {
let incoming = PeerId::random();
let ii = IncomingIndex(1);
let config = PeersetConfig {
in_peers: 50,
out_peers: 50,
bootnodes: vec![],
reserved_only: true,
priority_groups: vec![],
sets: vec![SetConfig {
in_peers: 50,
out_peers: 50,
bootnodes: vec![],
reserved_nodes: Default::default(),
reserved_only: true,
}],
};
let (mut peerset, _) = Peerset::from_config(config);
peerset.incoming(incoming, ii);
peerset.incoming(SetId::from(0), incoming.clone(), ii);
assert_messages(peerset, vec![
Message::Reject(ii),
@@ -787,32 +813,36 @@ mod tests {
let discovered = PeerId::random();
let discovered2 = PeerId::random();
let config = PeersetConfig {
in_peers: 0,
out_peers: 2,
bootnodes: vec![bootnode.clone()],
reserved_only: false,
priority_groups: vec![],
sets: vec![SetConfig {
in_peers: 0,
out_peers: 2,
bootnodes: vec![bootnode.clone()],
reserved_nodes: Default::default(),
reserved_only: false,
}],
};
let (mut peerset, _handle) = Peerset::from_config(config);
peerset.discovered(Some(discovered.clone()));
peerset.discovered(Some(discovered.clone()));
peerset.discovered(Some(discovered2));
peerset.add_to_peers_set(SetId::from(0), discovered.clone());
peerset.add_to_peers_set(SetId::from(0), discovered.clone());
peerset.add_to_peers_set(SetId::from(0), discovered2);
assert_messages(peerset, vec![
Message::Connect(bootnode),
Message::Connect(discovered),
Message::Connect { set_id: SetId::from(0), peer_id: bootnode },
Message::Connect { set_id: SetId::from(0), peer_id: discovered },
]);
}
#[test]
fn test_peerset_banned() {
let (mut peerset, handle) = Peerset::from_config(PeersetConfig {
in_peers: 25,
out_peers: 25,
bootnodes: vec![],
reserved_only: false,
priority_groups: vec![],
sets: vec![SetConfig {
in_peers: 25,
out_peers: 25,
bootnodes: vec![],
reserved_nodes: Default::default(),
reserved_only: false,
}],
});
// We ban a node by setting its reputation under the threshold.
@@ -824,7 +854,7 @@ mod tests {
assert_eq!(Stream::poll_next(Pin::new(&mut peerset), cx), Poll::Pending);
// Check that an incoming connection from that node gets refused.
peerset.incoming(peer_id.clone(), IncomingIndex(1));
peerset.incoming(SetId::from(0), peer_id.clone(), IncomingIndex(1));
if let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
assert_eq!(msg.unwrap(), Message::Reject(IncomingIndex(1)));
} else {
@@ -835,7 +865,7 @@ mod tests {
thread::sleep(Duration::from_millis(1500));
// Try again. This time the node should be accepted.
peerset.incoming(peer_id.clone(), IncomingIndex(2));
peerset.incoming(SetId::from(0), peer_id.clone(), IncomingIndex(2));
while let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) {
assert_eq!(msg.unwrap(), Message::Accept(IncomingIndex(2)));
}
substrate/client/peerset/src/peersstate.rs
+441 -167
View File
@@ -19,9 +19,10 @@
//! Reputation and slots allocation system behind the peerset.
//!
//! The [`PeersState`] state machine is responsible for managing the reputation and allocating
//! slots. It holds a list of nodes, each associated with a reputation value and whether we are
//! connected or not to this node. Thanks to this list, it knows how many slots are occupied. It
//! also holds a list of nodes which don't occupy slots.
//! slots. It holds a list of nodes, each associated with a reputation value, a list of sets the
//! node belongs to, and for each set whether we are connected or not to this node. Thanks to this
//! list, it knows how many slots are occupied. It also holds a list of nodes which don't occupy
//! slots.
//!
//! > Note: This module is purely dedicated to managing slots and reputations. Features such as
//! > for example connecting to some nodes in priority should be added outside of this
@@ -29,7 +30,10 @@
use libp2p::PeerId;
use log::error;
use std::{borrow::Cow, collections::{HashSet, HashMap}};
use std::{
borrow::Cow,
collections::{HashMap, HashSet, hash_map::{Entry, OccupiedEntry}},
};
use wasm_timer::Instant;
/// State storage behind the peerset.
@@ -48,16 +52,33 @@ pub struct PeersState {
/// sort, to make the logic easier.
nodes: HashMap<PeerId, Node>,
/// Number of slot-occupying nodes for which the `ConnectionState` is `In`.
/// Configuration of each set. The size of this `Vec` is never modified.
sets: Vec<SetInfo>,
}
/// Configuration of a single set.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct SetConfig {
/// Maximum allowed number of slot-occupying nodes for ingoing connections.
pub in_peers: u32,
/// Maximum allowed number of slot-occupying nodes for outgoing connections.
pub out_peers: u32,
}
/// State of a single set.
#[derive(Debug, Clone, PartialEq, Eq)]
struct SetInfo {
/// Number of slot-occupying nodes for which the `MembershipState` is `In`.
num_in: u32,
/// Number of slot-occupying nodes for which the `ConnectionState` is `In`.
/// Number of slot-occupying nodes for which the `MembershipState` is `In`.
num_out: u32,
/// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `In`.
/// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `In`.
max_in: u32,
/// Maximum allowed number of slot-occupying nodes for which the `ConnectionState` is `Out`.
/// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `Out`.
max_out: u32,
/// List of node identities (discovered or not) that don't occupy slots.
@@ -69,35 +90,37 @@ pub struct PeersState {
}
/// State of a single node that we know about.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
struct Node {
/// Whether we are connected to this node.
connection_state: ConnectionState,
/// List of sets the node belongs to.
/// Always has a fixed size equal to the one of [`PeersState::set`]. The various possible sets
/// are indices into this `Vec`.
sets: Vec<MembershipState>,
/// Reputation value of the node, between `i32::min_value` (we hate that node) and
/// `i32::max_value` (we love that node).
reputation: i32,
}
impl Default for Node {
fn default() -> Node {
impl Node {
fn new(num_sets: usize) -> Node {
Node {
connection_state: ConnectionState::NotConnected {
last_connected: Instant::now(),
},
sets: (0..num_sets).map(|_| MembershipState::NotMember).collect(),
reputation: 0,
}
}
}
/// Whether we are connected to a node.
/// Whether we are connected to a node in the context of a specific set.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum ConnectionState {
enum MembershipState {
/// Node isn't part of that set.
NotMember,
/// We are connected through an ingoing connection.
In,
/// We are connected through an outgoing connection.
Out,
/// We are not connected to this node.
/// Node is part of that set, but we are not connected to it.
NotConnected {
/// When we were last connected to the node, or if we were never connected when we
/// discovered it.
@@ -105,50 +128,87 @@ enum ConnectionState {
},
}
impl ConnectionState {
impl MembershipState {
/// Returns `true` for `In` and `Out`.
fn is_connected(self) -> bool {
match self {
ConnectionState::In => true,
ConnectionState::Out => true,
ConnectionState::NotConnected { .. } => false,
MembershipState::NotMember => false,
MembershipState::In => true,
MembershipState::Out => true,
MembershipState::NotConnected { .. } => false,
}
}
}
impl PeersState {
/// Builds a new empty `PeersState`.
pub fn new(in_peers: u32, out_peers: u32) -> Self {
pub fn new(sets: impl IntoIterator<Item = SetConfig>) -> Self {
PeersState {
nodes: HashMap::new(),
num_in: 0,
num_out: 0,
max_in: in_peers,
max_out: out_peers,
no_slot_nodes: HashSet::new(),
sets: sets
.into_iter()
.map(|config| SetInfo {
num_in: 0,
num_out: 0,
max_in: config.in_peers,
max_out: config.out_peers,
no_slot_nodes: HashSet::new(),
})
.collect(),
}
}
/// Returns an object that grants access to the state of a peer.
pub fn peer<'a>(&'a mut self, peer_id: &'a PeerId) -> Peer<'a> {
match self.nodes.get_mut(peer_id) {
None => Peer::Unknown(UnknownPeer {
/// Returns the number of sets.
///
/// Corresponds to the number of elements passed to [`PeersState::new`].
pub fn num_sets(&self) -> usize {
self.sets.len()
}
/// Returns an object that grants access to the reputation value of a peer.
pub fn peer_reputation(&mut self, peer_id: PeerId) -> Reputation {
if !self.nodes.contains_key(&peer_id) {
self.nodes.insert(peer_id.clone(), Node::new(self.sets.len()));
}
let entry = match self.nodes.entry(peer_id) {
Entry::Vacant(_) => unreachable!("guaranteed to be inserted above; qed"),
Entry::Occupied(e) => e,
};
Reputation { node: Some(entry) }
}
/// Returns an object that grants access to the state of a peer in the context of a specific
/// set.
///
/// # Panic
///
/// `set` must be within range of the sets passed to [`PeersState::new`].
///
pub fn peer<'a>(&'a mut self, set: usize, peer_id: &'a PeerId) -> Peer<'a> {
// The code below will panic anyway if this happens to be false, but this earlier assert
// makes it explicit what is wrong.
assert!(set < self.sets.len());
match self.nodes.get_mut(peer_id).map(|p| &p.sets[set]) {
None | Some(MembershipState::NotMember) => Peer::Unknown(UnknownPeer {
parent: self,
set,
peer_id: Cow::Borrowed(peer_id),
}),
Some(peer) => {
if peer.connection_state.is_connected() {
Peer::Connected(ConnectedPeer {
state: self,
peer_id: Cow::Borrowed(peer_id),
})
} else {
Peer::NotConnected(NotConnectedPeer {
state: self,
peer_id: Cow::Borrowed(peer_id),
})
}
Some(MembershipState::In) | Some(MembershipState::Out) => {
Peer::Connected(ConnectedPeer {
state: self,
set,
peer_id: Cow::Borrowed(peer_id),
})
}
Some(MembershipState::NotConnected { .. }) => Peer::NotConnected(NotConnectedPeer {
state: self,
set,
peer_id: Cow::Borrowed(peer_id),
}),
}
}
@@ -159,22 +219,49 @@ impl PeersState {
self.nodes.keys()
}
/// Returns the list of peers we are connected to.
/// Returns the list of peers we are connected to in the context of a specific set.
///
/// # Panic
///
/// `set` must be within range of the sets passed to [`PeersState::new`].
///
// Note: this method could theoretically return a `ConnectedPeer`, but implementing that
// isn't simple.
pub fn connected_peers(&self) -> impl Iterator<Item = &PeerId> {
self.nodes.iter()
.filter(|(_, p)| p.connection_state.is_connected())
pub fn connected_peers(&self, set: usize) -> impl Iterator<Item = &PeerId> {
// The code below will panic anyway if this happens to be false, but this earlier assert
// makes it explicit what is wrong.
assert!(set < self.sets.len());
self.nodes
.iter()
.filter(move |(_, p)| p.sets[set].is_connected())
.map(|(p, _)| p)
}
/// Returns the peer with the highest reputation and that we are not connected to.
///
/// If multiple nodes have the same reputation, which one is returned is unspecified.
pub fn highest_not_connected_peer(&mut self) -> Option<NotConnectedPeer> {
let outcome = self.nodes
///
/// # Panic
///
/// `set` must be within range of the sets passed to [`PeersState::new`].
///
pub fn highest_not_connected_peer(&mut self, set: usize) -> Option<NotConnectedPeer> {
// The code below will panic anyway if this happens to be false, but this earlier assert
// makes it explicit what is wrong.
assert!(set < self.sets.len());
let outcome = self
.nodes
.iter_mut()
.filter(|(_, Node { connection_state, .. })| !connection_state.is_connected())
.filter(|(_, Node { sets, .. })| {
match sets[set] {
MembershipState::NotMember => false,
MembershipState::In => false,
MembershipState::Out => false,
MembershipState::NotConnected { .. } => true,
}
})
.fold(None::<(&PeerId, &mut Node)>, |mut cur_node, to_try| {
if let Some(cur_node) = cur_node.take() {
if cur_node.1.reputation >= to_try.1.reputation {
@@ -188,6 +275,7 @@ impl PeersState {
if let Some(peer_id) = outcome {
Some(NotConnectedPeer {
state: self,
set,
peer_id: Cow::Owned(peer_id),
})
} else {
@@ -197,48 +285,48 @@ impl PeersState {
/// Add a node to the list of nodes that don't occupy slots.
///
/// Has no effect if the peer was already in the group.
pub fn add_no_slot_node(&mut self, peer_id: PeerId) {
/// Has no effect if the node was already in the group.
pub fn add_no_slot_node(&mut self, set: usize, peer_id: PeerId) {
// Reminder: `HashSet::insert` returns false if the node was already in the set
if !self.no_slot_nodes.insert(peer_id.clone()) {
if !self.sets[set].no_slot_nodes.insert(peer_id.clone()) {
return;
}
if let Some(peer) = self.nodes.get_mut(&peer_id) {
match peer.connection_state {
ConnectionState::In => self.num_in -= 1,
ConnectionState::Out => self.num_out -= 1,
ConnectionState::NotConnected { .. } => {},
match peer.sets[set] {
MembershipState::In => self.sets[set].num_in -= 1,
MembershipState::Out => self.sets[set].num_out -= 1,
MembershipState::NotConnected { .. } | MembershipState::NotMember => {}
}
}
}
/// Removes a node from the list of nodes that don't occupy slots.
///
/// Has no effect if the peer was not in the group.
pub fn remove_no_slot_node(&mut self, peer_id: &PeerId) {
/// Has no effect if the node was not in the group.
pub fn remove_no_slot_node(&mut self, set: usize, peer_id: &PeerId) {
// Reminder: `HashSet::remove` returns false if the node was already not in the set
if !self.no_slot_nodes.remove(peer_id) {
if !self.sets[set].no_slot_nodes.remove(peer_id) {
return;
}
if let Some(peer) = self.nodes.get_mut(peer_id) {
match peer.connection_state {
ConnectionState::In => self.num_in += 1,
ConnectionState::Out => self.num_out += 1,
ConnectionState::NotConnected { .. } => {},
match peer.sets[set] {
MembershipState::In => self.sets[set].num_in += 1,
MembershipState::Out => self.sets[set].num_out += 1,
MembershipState::NotConnected { .. } | MembershipState::NotMember => {}
}
}
}
}
/// Grants access to the state of a peer in the `PeersState`.
/// Grants access to the state of a peer in the [`PeersState`] in the context of a specific set.
pub enum Peer<'a> {
/// We are connected to this node.
Connected(ConnectedPeer<'a>),
/// We are not connected to this node.
NotConnected(NotConnectedPeer<'a>),
/// We have never heard of this node.
/// We have never heard of this node, or it is not part of the set.
Unknown(UnknownPeer<'a>),
}
@@ -255,7 +343,7 @@ impl<'a> Peer<'a> {
/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
/// otherwise.
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
pub fn into_not_connected(self) -> Option<NotConnectedPeer<'a>> {
match self {
Peer::Connected(_) => None,
@@ -266,7 +354,7 @@ impl<'a> Peer<'a> {
/// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None`
/// otherwise.
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
pub fn into_unknown(self) -> Option<UnknownPeer<'a>> {
match self {
Peer::Connected(_) => None,
@@ -279,10 +367,16 @@ impl<'a> Peer<'a> {
/// A peer that is connected to us.
pub struct ConnectedPeer<'a> {
state: &'a mut PeersState,
set: usize,
peer_id: Cow<'a, PeerId>,
}
impl<'a> ConnectedPeer<'a> {
/// Get the `PeerId` associated to this `ConnectedPeer`.
pub fn peer_id(&self) -> &PeerId {
&self.peer_id
}
/// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it.
pub fn into_peer_id(self) -> PeerId {
self.peer_id.into_owned()
@@ -290,65 +384,74 @@ impl<'a> ConnectedPeer<'a> {
/// Switches the peer to "not connected".
pub fn disconnect(self) -> NotConnectedPeer<'a> {
let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
if let Some(mut node) = self.state.nodes.get_mut(&*self.peer_id) {
let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
if !is_no_slot_occupy {
match node.connection_state {
ConnectionState::In => self.state.num_in -= 1,
ConnectionState::Out => self.state.num_out -= 1,
ConnectionState::NotConnected { .. } =>
debug_assert!(false, "State inconsistency: disconnecting a disconnected node")
match node.sets[self.set] {
MembershipState::In => self.state.sets[self.set].num_in -= 1,
MembershipState::Out => self.state.sets[self.set].num_out -= 1,
MembershipState::NotMember | MembershipState::NotConnected { .. } => {
debug_assert!(
false,
"State inconsistency: disconnecting a disconnected node"
)
}
}
}
node.connection_state = ConnectionState::NotConnected {
node.sets[self.set] = MembershipState::NotConnected {
last_connected: Instant::now(),
};
} else {
debug_assert!(false, "State inconsistency: disconnecting a disconnected node");
debug_assert!(
false,
"State inconsistency: disconnecting a disconnected node"
);
}
NotConnectedPeer {
state: self.state,
set: self.set,
peer_id: self.peer_id,
}
}
/// Returns the reputation value of the node.
pub fn reputation(&self) -> i32 {
self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation)
}
/// Sets the reputation of the peer.
pub fn set_reputation(&mut self, value: i32) {
if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
node.reputation = value;
} else {
debug_assert!(false, "State inconsistency: set_reputation on an unknown node");
}
}
/// Performs an arithmetic addition on the reputation score of that peer.
///
/// In case of overflow, the value will be capped.
///
/// > **Note**: Reputation values aren't specific to a set but are global per peer.
pub fn add_reputation(&mut self, modifier: i32) {
if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
node.reputation = node.reputation.saturating_add(modifier);
} else {
debug_assert!(false, "State inconsistency: add_reputation on an unknown node");
debug_assert!(
false,
"State inconsistency: add_reputation on an unknown node"
);
}
}
/// Returns the reputation value of the node.
///
/// > **Note**: Reputation values aren't specific to a set but are global per peer.
pub fn reputation(&self) -> i32 {
self.state
.nodes
.get(&*self.peer_id)
.map_or(0, |p| p.reputation)
}
}
/// A peer that is not connected to us.
#[derive(Debug)]
pub struct NotConnectedPeer<'a> {
state: &'a mut PeersState,
set: usize,
peer_id: Cow<'a, PeerId>,
}
impl<'a> NotConnectedPeer<'a> {
/// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it.
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
pub fn into_peer_id(self) -> PeerId {
self.peer_id.into_owned()
}
@@ -361,7 +464,7 @@ impl<'a> NotConnectedPeer<'a> {
None => return,
};
if let ConnectionState::NotConnected { last_connected } = &mut state.connection_state {
if let MembershipState::NotConnected { last_connected } = &mut state.sets[self.set] {
*last_connected = Instant::now();
}
}
@@ -383,8 +486,8 @@ impl<'a> NotConnectedPeer<'a> {
}
};
match state.connection_state {
ConnectionState::NotConnected { last_connected } => last_connected,
match state.sets[self.set] {
MembershipState::NotConnected { last_connected } => last_connected,
_ => {
error!(target: "peerset", "State inconsistency with {}", self.peer_id);
Instant::now()
@@ -399,25 +502,31 @@ impl<'a> NotConnectedPeer<'a> {
///
/// Non-slot-occupying nodes don't count towards the number of slots.
pub fn try_outgoing(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
// Note that it is possible for num_out to be strictly superior to the max, in case we were
// connected to reserved node then marked them as not reserved.
if self.state.num_out >= self.state.max_out && !is_no_slot_occupy {
if self.state.sets[self.set].num_out >= self.state.sets[self.set].max_out
&& !is_no_slot_occupy
{
return Err(self);
}
if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) {
peer.connection_state = ConnectionState::Out;
if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
peer.sets[self.set] = MembershipState::Out;
if !is_no_slot_occupy {
self.state.num_out += 1;
self.state.sets[self.set].num_out += 1;
}
} else {
debug_assert!(false, "State inconsistency: try_outgoing on an unknown node");
debug_assert!(
false,
"State inconsistency: try_outgoing on an unknown node"
);
}
Ok(ConnectedPeer {
state: self.state,
set: self.set,
peer_id: self.peer_id,
})
}
@@ -429,74 +538,95 @@ impl<'a> NotConnectedPeer<'a> {
///
/// Non-slot-occupying nodes don't count towards the number of slots.
pub fn try_accept_incoming(self) -> Result<ConnectedPeer<'a>, NotConnectedPeer<'a>> {
let is_no_slot_occupy = self.state.no_slot_nodes.contains(&*self.peer_id);
let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id);
// Note that it is possible for num_in to be strictly superior to the max, in case we were
// connected to reserved node then marked them as not reserved.
if self.state.num_in >= self.state.max_in && !is_no_slot_occupy {
if self.state.sets[self.set].num_in >= self.state.sets[self.set].max_in
&& !is_no_slot_occupy
{
return Err(self);
}
if let Some(mut peer) = self.state.nodes.get_mut(&*self.peer_id) {
peer.connection_state = ConnectionState::In;
if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
peer.sets[self.set] = MembershipState::In;
if !is_no_slot_occupy {
self.state.num_in += 1;
self.state.sets[self.set].num_in += 1;
}
} else {
debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node");
debug_assert!(
false,
"State inconsistency: try_accept_incoming on an unknown node"
);
}
Ok(ConnectedPeer {
state: self.state,
set: self.set,
peer_id: self.peer_id,
})
}
/// Returns the reputation value of the node.
///
/// > **Note**: Reputation values aren't specific to a set but are global per peer.
pub fn reputation(&self) -> i32 {
self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation)
self.state
.nodes
.get(&*self.peer_id)
.map_or(0, |p| p.reputation)
}
/// Sets the reputation of the peer.
///
/// > **Note**: Reputation values aren't specific to a set but are global per peer.
#[cfg(test)] // Feel free to remove this if this function is needed outside of tests
pub fn set_reputation(&mut self, value: i32) {
if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
node.reputation = value;
} else {
debug_assert!(false, "State inconsistency: set_reputation on an unknown node");
debug_assert!(
false,
"State inconsistency: set_reputation on an unknown node"
);
}
}
/// Performs an arithmetic addition on the reputation score of that peer.
///
/// In case of overflow, the value will be capped.
pub fn add_reputation(&mut self, modifier: i32) {
if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) {
node.reputation = node.reputation.saturating_add(modifier);
} else {
debug_assert!(false, "State inconsistency: add_reputation on an unknown node");
}
}
/// Un-discovers the peer. Removes it from the list.
/// Removes the peer from the list of members of the set.
pub fn forget_peer(self) -> UnknownPeer<'a> {
if self.state.nodes.remove(&*self.peer_id).is_none() {
if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) {
debug_assert!(!matches!(peer.sets[self.set], MembershipState::NotMember));
peer.sets[self.set] = MembershipState::NotMember;
// Remove the peer from `self.state.nodes` entirely if it isn't a member of any set.
if peer.reputation == 0 && peer
.sets
.iter()
.all(|set| matches!(set, MembershipState::NotMember))
{
self.state.nodes.remove(&*self.peer_id);
}
} else {
debug_assert!(false, "State inconsistency: forget_peer on an unknown node");
error!(
target: "peerset",
"State inconsistency with {} when forgetting peer",
self.peer_id
);
}
};
UnknownPeer {
parent: self.state,
set: self.set,
peer_id: self.peer_id,
}
}
}
/// A peer that we have never heard of.
/// A peer that we have never heard of or that isn't part of the set.
pub struct UnknownPeer<'a> {
parent: &'a mut PeersState,
set: usize,
peer_id: Cow<'a, PeerId>,
}
@@ -506,96 +636,240 @@ impl<'a> UnknownPeer<'a> {
/// The node starts with a reputation of 0. You can adjust these default
/// values using the `NotConnectedPeer` that this method returns.
pub fn discover(self) -> NotConnectedPeer<'a> {
self.parent.nodes.insert(self.peer_id.clone().into_owned(), Node {
connection_state: ConnectionState::NotConnected {
last_connected: Instant::now(),
},
reputation: 0,
});
let num_sets = self.parent.sets.len();
self.parent
.nodes
.entry(self.peer_id.clone().into_owned())
.or_insert_with(|| Node::new(num_sets))
.sets[self.set] = MembershipState::NotConnected {
last_connected: Instant::now(),
};
let state = self.parent;
NotConnectedPeer {
state,
state: self.parent,
set: self.set,
peer_id: self.peer_id,
}
}
}
/// Access to the reputation of a peer.
pub struct Reputation<'a> {
/// Node entry in [`PeersState::nodes`]. Always `Some` except right before dropping.
node: Option<OccupiedEntry<'a, PeerId, Node>>,
}
impl<'a> Reputation<'a> {
/// Returns the reputation value of the node.
pub fn reputation(&self) -> i32 {
self.node.as_ref().unwrap().get().reputation
}
/// Sets the reputation of the peer.
pub fn set_reputation(&mut self, value: i32) {
self.node.as_mut().unwrap().get_mut().reputation = value;
}
/// Performs an arithmetic addition on the reputation score of that peer.
///
/// In case of overflow, the value will be capped.
pub fn add_reputation(&mut self, modifier: i32) {
let reputation = &mut self.node.as_mut().unwrap().get_mut().reputation;
*reputation = reputation.saturating_add(modifier);
}
}
impl<'a> Drop for Reputation<'a> {
fn drop(&mut self) {
if let Some(node) = self.node.take() {
if node.get().reputation == 0 &&
node.get().sets.iter().all(|set| matches!(set, MembershipState::NotMember))
{
node.remove();
}
}
}
}
#[cfg(test)]
mod tests {
use super::{PeersState, Peer};
use super::{Peer, PeersState, SetConfig};
use libp2p::PeerId;
use std::iter;
#[test]
fn full_slots_in() {
let mut peers_state = PeersState::new(1, 1);
let mut peers_state = PeersState::new(iter::once(SetConfig {
in_peers: 1,
out_peers: 1,
}));
let id1 = PeerId::random();
let id2 = PeerId::random();
if let Peer::Unknown(e) = peers_state.peer(&id1) {
if let Peer::Unknown(e) = peers_state.peer(0, &id1) {
assert!(e.discover().try_accept_incoming().is_ok());
}
if let Peer::Unknown(e) = peers_state.peer(&id2) {
if let Peer::Unknown(e) = peers_state.peer(0, &id2) {
assert!(e.discover().try_accept_incoming().is_err());
}
}
#[test]
fn no_slot_node_doesnt_use_slot() {
let mut peers_state = PeersState::new(1, 1);
let mut peers_state = PeersState::new(iter::once(SetConfig {
in_peers: 1,
out_peers: 1,
}));
let id1 = PeerId::random();
let id2 = PeerId::random();
peers_state.add_no_slot_node(id1.clone());
if let Peer::Unknown(p) = peers_state.peer(&id1) {
peers_state.add_no_slot_node(0, id1.clone());
if let Peer::Unknown(p) = peers_state.peer(0, &id1) {
assert!(p.discover().try_accept_incoming().is_ok());
} else { panic!() }
} else {
panic!()
}
if let Peer::Unknown(e) = peers_state.peer(&id2) {
if let Peer::Unknown(e) = peers_state.peer(0, &id2) {
assert!(e.discover().try_accept_incoming().is_ok());
} else { panic!() }
} else {
panic!()
}
}
#[test]
fn disconnecting_frees_slot() {
let mut peers_state = PeersState::new(1, 1);
let mut peers_state = PeersState::new(iter::once(SetConfig {
in_peers: 1,
out_peers: 1,
}));
let id1 = PeerId::random();
let id2 = PeerId::random();
assert!(peers_state.peer(&id1).into_unknown().unwrap().discover().try_accept_incoming().is_ok());
assert!(peers_state.peer(&id2).into_unknown().unwrap().discover().try_accept_incoming().is_err());
peers_state.peer(&id1).into_connected().unwrap().disconnect();
assert!(peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().is_ok());
assert!(peers_state
.peer(0, &id1)
.into_unknown()
.unwrap()
.discover()
.try_accept_incoming()
.is_ok());
assert!(peers_state
.peer(0, &id2)
.into_unknown()
.unwrap()
.discover()
.try_accept_incoming()
.is_err());
peers_state
.peer(0, &id1)
.into_connected()
.unwrap()
.disconnect();
assert!(peers_state
.peer(0, &id2)
.into_not_connected()
.unwrap()
.try_accept_incoming()
.is_ok());
}
#[test]
fn highest_not_connected_peer() {
let mut peers_state = PeersState::new(25, 25);
let mut peers_state = PeersState::new(iter::once(SetConfig {
in_peers: 25,
out_peers: 25,
}));
let id1 = PeerId::random();
let id2 = PeerId::random();
assert!(peers_state.highest_not_connected_peer().is_none());
peers_state.peer(&id1).into_unknown().unwrap().discover().set_reputation(50);
peers_state.peer(&id2).into_unknown().unwrap().discover().set_reputation(25);
assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
peers_state.peer(&id2).into_not_connected().unwrap().set_reputation(75);
assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2.clone()));
peers_state.peer(&id2).into_not_connected().unwrap().try_accept_incoming().unwrap();
assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(100);
peers_state.peer(&id2).into_connected().unwrap().disconnect();
assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id1.clone()));
peers_state.peer(&id1).into_not_connected().unwrap().set_reputation(-100);
assert_eq!(peers_state.highest_not_connected_peer().map(|p| p.into_peer_id()), Some(id2));
assert!(peers_state.highest_not_connected_peer(0).is_none());
peers_state
.peer(0, &id1)
.into_unknown()
.unwrap()
.discover()
.set_reputation(50);
peers_state
.peer(0, &id2)
.into_unknown()
.unwrap()
.discover()
.set_reputation(25);
assert_eq!(
peers_state
.highest_not_connected_peer(0)
.map(|p| p.into_peer_id()),
Some(id1.clone())
);
peers_state
.peer(0, &id2)
.into_not_connected()
.unwrap()
.set_reputation(75);
assert_eq!(
peers_state
.highest_not_connected_peer(0)
.map(|p| p.into_peer_id()),
Some(id2.clone())
);
peers_state
.peer(0, &id2)
.into_not_connected()
.unwrap()
.try_accept_incoming()
.unwrap();
assert_eq!(
peers_state
.highest_not_connected_peer(0)
.map(|p| p.into_peer_id()),
Some(id1.clone())
);
peers_state
.peer(0, &id1)
.into_not_connected()
.unwrap()
.set_reputation(100);
peers_state
.peer(0, &id2)
.into_connected()
.unwrap()
.disconnect();
assert_eq!(
peers_state
.highest_not_connected_peer(0)
.map(|p| p.into_peer_id()),
Some(id1.clone())
);
peers_state
.peer(0, &id1)
.into_not_connected()
.unwrap()
.set_reputation(-100);
assert_eq!(
peers_state
.highest_not_connected_peer(0)
.map(|p| p.into_peer_id()),
Some(id2.clone())
);
}
#[test]
fn disconnect_no_slot_doesnt_panic() {
let mut peers_state = PeersState::new(1, 1);
let mut peers_state = PeersState::new(iter::once(SetConfig {
in_peers: 1,
out_peers: 1,
}));
let id = PeerId::random();
peers_state.add_no_slot_node(id.clone());
let peer = peers_state.peer(&id).into_unknown().unwrap().discover().try_outgoing().unwrap();
peers_state.add_no_slot_node(0, id.clone());
let peer = peers_state
.peer(0, &id)
.into_unknown()
.unwrap()
.discover()
.try_outgoing()
.unwrap();
peer.disconnect();
}
}