// Copyright 2018 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see .?
use fnv::FnvHashMap;
use libp2p::{Multiaddr, PeerId, identify::IdentifyTopology, multihash::Multihash};
use libp2p::core::{PublicKey, swarm::ConnectedPoint, topology::DisconnectReason, topology::Topology};
use libp2p::kad::{KBucketsPeerId, KadConnectionType, KademliaTopology};
use log::{debug, info, trace, warn};
use serde_derive::{Serialize, Deserialize};
use std::{cmp, fs, iter, vec};
use std::io::{Read, Cursor, Error as IoError, ErrorKind as IoErrorKind, Write, BufReader, BufWriter};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime};
/// For each address we're connected to, a period of this duration increases the score by 1.
const CONNEC_DURATION_PER_SCORE: Duration = Duration::from_secs(10);
/// Maximum number of addresses for a given peer. If there are more than this number of addresses,
/// the ones with a lower score will be removed.
const MAX_ADDRESSES_PER_PEER: usize = 10;
/// Maximum value for the score.
const MAX_SCORE: u32 = 100;
/// When we successfully connect to a node, raises its score to the given minimum value.
const CONNECTED_MINIMUM_SCORE: u32 = 20;
/// Initial score that a node discovered through Kademlia receives, where we have a hint that the
/// node is reachable.
const DISCOVERY_INITIAL_SCORE_CONNECTABLE: u32 = 15;
/// Initial score that a node discovered through Kademlia receives, without any hint.
const DISCOVERY_INITIAL_SCORE: u32 = 10;
/// Score adjustement when we fail to connect to an address.
const SCORE_DIFF_ON_FAILED_TO_CONNECT: i32 = -1;
/// Default time-to-live for addresses discovered through Kademlia.
/// After this time has elapsed and no connection has succeeded, the address will be removed.
const KADEMLIA_DISCOVERY_EXPIRATION: Duration = Duration::from_secs(2 * 3600);
/// After a successful connection, the TTL is set to a minimum at this amount.
const EXPIRATION_PUSH_BACK_CONNEC: Duration = Duration::from_secs(2 * 3600);
/// Initial score that a bootstrap node receives when registered.
const BOOTSTRAP_NODE_SCORE: u32 = 100;
/// Time to live of a boostrap node. This only applies if you start the node later *without*
/// that bootstrap node configured anymore.
const BOOTSTRAP_NODE_EXPIRATION: Duration = Duration::from_secs(24 * 3600);
/// The first time we fail to connect to an address, wait this duration before trying again.
const FIRST_CONNECT_FAIL_BACKOFF: Duration = Duration::from_secs(2);
/// Every time we fail to connect to an address, multiply the backoff by this constant.
const FAIL_BACKOFF_MULTIPLIER: u32 = 2;
/// We need a maximum value for the backoff, overwise we risk an overflow.
const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
// TODO: should be merged with the Kademlia k-buckets
/// Stores information about the topology of the network.
#[derive(Debug)]
pub struct NetTopology {
/// The actual storage. Never contains a key for `local_peer_id`.
store: FnvHashMap,
/// Optional path to the file that caches the serialized version of `store`.
cache_path: Option,
/// Public key of the local node.
local_public_key: PublicKey,
/// PeerId of the local node. Derived from `local_public_key`.
local_peer_id: PeerId,
/// Known addresses for the local node to report to the network.
external_addresses: Vec,
}
impl NetTopology {
/// Initializes a new `NetTopology` that isn't tied to any file.
///
/// `flush_to_disk()` will be a no-op.
#[inline]
pub fn memory(local_public_key: PublicKey) -> NetTopology {
let local_peer_id = local_public_key.clone().into_peer_id();
NetTopology {
store: Default::default(),
cache_path: None,
local_peer_id,
local_public_key,
external_addresses: Vec::new(),
}
}
/// Builds a `NetTopology` that will use `path` as a cache.
///
/// This function tries to load a known topology from the file. If the file doesn't exist
/// or contains garbage data, the execution still continues.
///
/// Calling `flush_to_disk()` in the future writes to the given path.
pub fn from_file>(local_public_key: PublicKey, path: P) -> NetTopology {
let path = path.as_ref();
let local_peer_id = local_public_key.clone().into_peer_id();
debug!(target: "sub-libp2p", "Initializing peer store for JSON file {:?}", path);
let store = try_load(path, &local_peer_id);
NetTopology {
store,
cache_path: Some(path.to_owned()),
local_peer_id,
local_public_key,
external_addresses: Vec::new(),
}
}
/// Writes the topology into the path passed to `from_file`.
///
/// No-op if the object was created with `memory()`.
pub fn flush_to_disk(&mut self) -> Result<(), IoError> {
let path = match self.cache_path {
Some(ref p) => p,
None => return Ok(())
};
let file = fs::File::create(path)?;
// TODO: the capacity of the BufWriter is kind of arbitrary ; decide better
serialize(BufWriter::with_capacity(1024 * 1024, file), &mut self.store)
}
/// Returns the number of peers in the topology, excluding the local peer.
#[inline]
pub fn num_peers(&self) -> usize {
self.store.len()
}
/// Perform a cleanup pass, removing all obsolete addresses and peers.
///
/// This should be done from time to time.
pub fn cleanup(&mut self) {
let now_systime = SystemTime::now();
self.store.retain(|_, peer| {
let new_addrs = peer.addrs
.drain(..)
.filter(|a| a.expires > now_systime || a.is_connected())
.collect();
peer.addrs = new_addrs;
!peer.addrs.is_empty()
});
}
/// Add the external addresses that are known for the local node.
pub fn add_external_addrs(&mut self, addrs: TIter)
where TIter: Iterator- {
self.external_addresses.extend(addrs);
}
/// Returns a list of all the known addresses of peers, ordered by the
/// order in which we should attempt to connect to them.
///
/// Because of expiration and back-off mechanisms, this list can grow
/// by itself over time. The `Instant` that is returned corresponds to
/// the earlier known time when a new entry will be added automatically to
/// the list.
pub fn addrs_to_attempt(&mut self) -> (impl Iterator
- , Instant) {
// TODO: optimize
let now = Instant::now();
let now_systime = SystemTime::now();
let mut instant = now + Duration::from_secs(3600);
let mut addrs_out = Vec::new();
let mut peer_addrs = Vec::new();
'peer_loop: for (peer, info) in &mut self.store {
peer_addrs.clear();
for addr in &mut info.addrs {
let (score, is_connected) = addr.score_and_is_connected();
if is_connected {
continue 'peer_loop
}
if score == 0 || addr.expires < now_systime {
continue
}
if addr.back_off_until > now {
instant = cmp::min(instant, addr.back_off_until);
continue
}
peer_addrs.push(((peer, &addr.addr), score));
}
for val in peer_addrs.drain(..) {
addrs_out.push(val);
}
}
addrs_out.sort_by(|a, b| b.1.cmp(&a.1));
(addrs_out.into_iter().map(|a| a.0), instant)
}
/// Adds an address corresponding to a boostrap node.
///
/// We assume that the address is valid, so its score starts very high.
pub fn add_bootstrap_addr(&mut self, peer: &PeerId, addr: Multiaddr) {
let now_systime = SystemTime::now();
let now = Instant::now();
let peer = peer_access(&mut self.store, peer);
let mut found = false;
let new_addrs = peer.addrs
.drain(..)
.filter_map(|a| {
if a.expires < now_systime && !a.is_connected() {
return None
}
if a.addr == addr {
found = true;
}
Some(a)
})
.collect();
peer.addrs = new_addrs;
if !found {
peer.addrs.push(Addr {
addr,
expires: now_systime + BOOTSTRAP_NODE_EXPIRATION,
back_off_until: now,
next_back_off: FIRST_CONNECT_FAIL_BACKOFF,
score: AddrScore {
connected_since: None,
score: BOOTSTRAP_NODE_SCORE,
latest_score_update: now,
},
});
}
}
/// Inner implementaiton of the `add_*_discovered_addrs` methods.
/// Returns `true` if the topology has changed in some way. Returns `false` if calling this
/// method was a no-op.
fn add_discovered_addrs(
&mut self,
peer_id: &PeerId,
addrs: I,
) -> bool
where I: Iterator
- {
let mut addrs: Vec<_> = addrs.collect();
let now_systime = SystemTime::now();
let now = Instant::now();
let peer = peer_access(&mut self.store, peer_id);
let new_addrs = peer.addrs
.drain(..)
.filter_map(|a| {
if a.expires < now_systime && !a.is_connected() {
return None
}
if let Some(pos) = addrs.iter().position(|&(ref addr, _)| addr == &a.addr) {
addrs.remove(pos);
}
Some(a)
})
.collect();
peer.addrs = new_addrs;
let mut anything_changed = false;
if !addrs.is_empty() {
trace!(
target: "sub-libp2p",
"Peer store: adding addresses {:?} for {:?}",
addrs,
peer_id,
);
}
'addrs_inserter: for (addr, connectable) in addrs {
let initial_score = if connectable {
DISCOVERY_INITIAL_SCORE_CONNECTABLE
} else {
DISCOVERY_INITIAL_SCORE
};
// Enforce `MAX_ADDRESSES_PER_PEER` before inserting, or skip this entry.
while peer.addrs.len() >= MAX_ADDRESSES_PER_PEER {
let pos = peer.addrs.iter_mut().position(|addr| addr.score() <= initial_score);
if let Some(pos) = pos {
let _ = peer.addrs.remove(pos);
} else {
continue 'addrs_inserter;
}
}
anything_changed = true;
peer.addrs.push(Addr {
addr,
expires: now_systime + KADEMLIA_DISCOVERY_EXPIRATION,
back_off_until: now,
next_back_off: FIRST_CONNECT_FAIL_BACKOFF,
score: AddrScore {
connected_since: None,
score: initial_score,
latest_score_update: now,
},
});
}
anything_changed
}
}
impl KademliaTopology for NetTopology {
type ClosestPeersIter = vec::IntoIter;
type GetProvidersIter = iter::Empty;
fn add_kad_discovered_address(&mut self, peer: PeerId, addr: Multiaddr, ty: KadConnectionType) {
self.add_discovered_addrs(&peer, iter::once((addr, ty == KadConnectionType::Connected)));
}
fn closest_peers(&mut self, target: &Multihash, _max: usize) -> Self::ClosestPeersIter {
// TODO: very inefficient
let mut peers = self.store.keys().cloned().collect::>();
peers.push(self.local_peer_id.clone());
peers.sort_by(|a, b| {
b.as_ref().distance_with(target).cmp(&a.as_ref().distance_with(target))
});
peers.into_iter()
}
fn add_provider(&mut self, _: Multihash, _: PeerId) {
// We don't implement ADD_PROVIDER/GET_PROVIDERS
}
fn get_providers(&mut self, _: &Multihash) -> Self::GetProvidersIter {
// We don't implement ADD_PROVIDER/GET_PROVIDERS
iter::empty()
}
}
impl IdentifyTopology for NetTopology {
#[inline]
fn add_identify_discovered_addrs(&mut self, peer: &PeerId, addrs: TIter)
where
TIter: Iterator
-
{
// These are addresses that peers indicate for themselves.
// The typical use case is:
// - A peer connects to one of our listening points.
// - We send an identify request to it, and it answers with a list of addresses.
// - If later it disconnects, we can try to dial it back through one of these addresses.
self.add_discovered_addrs(peer, addrs.map(move |a| (a, true)));
}
}
impl Topology for NetTopology {
#[inline]
fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec {
if peer == &self.local_peer_id {
return self.external_addresses.clone()
}
let peer = if let Some(peer) = self.store.get_mut(peer) {
peer
} else {
return Vec::new()
};
let now_st = SystemTime::now();
let now_is = Instant::now();
let mut list = peer.addrs.iter_mut().filter_map(move |addr| {
let (score, connected) = addr.score_and_is_connected();
if (addr.expires >= now_st && score > 0 && addr.back_off_until < now_is) || connected {
Some((score, &addr.addr))
} else {
None
}
}).collect::>();
list.sort_by(|a, b| a.0.cmp(&b.0));
// TODO: meh, optimize
list.into_iter().map(|(_, addr)| addr.clone()).collect::>()
}
fn add_local_external_addrs(&mut self, addrs: TIter)
where TIter: Iterator
- {
self.add_external_addrs(addrs)
}
fn local_peer_id(&self) -> &PeerId {
&self.local_peer_id
}
fn local_public_key(&self) -> &PublicKey {
&self.local_public_key
}
fn set_connected(&mut self, peer: &PeerId, endpoint: &ConnectedPoint) {
let addr = match endpoint {
ConnectedPoint::Dialer { address } => address,
ConnectedPoint::Listener { .. } => return
};
let now = Instant::now();
// Just making sure that we have an entry for this peer in `store`, but don't use it.
let _ = peer_access(&mut self.store, peer);
for (peer_in_store, info_in_store) in self.store.iter_mut() {
if peer == peer_in_store {
if let Some(addr) = info_in_store.addrs.iter_mut().find(|a| &a.addr == addr) {
addr.connected_now(CONNECTED_MINIMUM_SCORE);
addr.back_off_until = now;
addr.next_back_off = FIRST_CONNECT_FAIL_BACKOFF;
continue
}
info_in_store.addrs.push(Addr {
addr: addr.clone(),
expires: SystemTime::now() + EXPIRATION_PUSH_BACK_CONNEC,
back_off_until: now,
next_back_off: FIRST_CONNECT_FAIL_BACKOFF,
score: AddrScore {
connected_since: Some(now),
latest_score_update: now,
score: CONNECTED_MINIMUM_SCORE,
},
});
} else {
// Set the score to 0 for any address that matches the one we connected to.
for addr_in_store in &mut info_in_store.addrs {
if &addr_in_store.addr == addr {
addr_in_store.adjust_score(-(MAX_SCORE as i32));
}
}
}
}
}
fn set_disconnected(&mut self, _: &PeerId, endpoint: &ConnectedPoint, reason: DisconnectReason) {
let addr = match endpoint {
ConnectedPoint::Dialer { address } => address,
ConnectedPoint::Listener { .. } => return
};
let score_diff = match reason {
DisconnectReason::Replaced => -3,
DisconnectReason::Graceful => -1,
DisconnectReason::Error => -5,
};
for info in self.store.values_mut() {
for a in info.addrs.iter_mut() {
if &a.addr == addr {
a.disconnected_now(score_diff);
a.back_off_until = Instant::now() + a.next_back_off;
a.next_back_off = cmp::min(a.next_back_off * FAIL_BACKOFF_MULTIPLIER, MAX_BACKOFF);
let expires_push_back = SystemTime::now() + EXPIRATION_PUSH_BACK_CONNEC;
if a.expires < expires_push_back {
a.expires = expires_push_back;
}
return
}
}
}
}
fn set_unreachable(&mut self, addr: &Multiaddr) {
for info in self.store.values_mut() {
for a in info.addrs.iter_mut() {
if &a.addr != addr {
continue
}
a.adjust_score(SCORE_DIFF_ON_FAILED_TO_CONNECT);
trace!(target: "sub-libp2p", "Back off for {} = {:?}", addr, a.next_back_off);
a.back_off_until = Instant::now() + a.next_back_off;
a.next_back_off = cmp::min(a.next_back_off * FAIL_BACKOFF_MULTIPLIER, MAX_BACKOFF);
}
}
}
}
fn peer_access<'a>(store: &'a mut FnvHashMap, peer: &PeerId) -> &'a mut PeerInfo {
// TODO: should be optimizable if HashMap gets a better API
store.entry(peer.clone()).or_insert_with(Default::default)
}
#[derive(Debug, Clone, Default)]
struct PeerInfo {
/// Addresses of that peer.
addrs: Vec,
}
#[derive(Debug)]
struct Addr {
/// The multiaddress.
addr: Multiaddr,
/// When the address expires.
expires: SystemTime,
next_back_off: Duration,
/// Don't try to connect to this node until `Instant`.
back_off_until: Instant,
score: AddrScore,
}
impl Clone for Addr {
fn clone(&self) -> Addr {
Addr {
addr: self.addr.clone(),
expires: self.expires,
next_back_off: self.next_back_off,
back_off_until: self.back_off_until,
score: self.score.clone(),
}
}
}
#[derive(Debug, Clone)]
struct AddrScore {
/// If connected, contains the moment when we connected. `None` if we're not connected.
connected_since: Option,
/// Score of this address. Potentially needs to be updated based on `latest_score_update`.
score: u32,
/// When we last updated the score.
latest_score_update: Instant,
}
impl Addr {
/// Sets the addr to connected. If the score is lower than the given value, raises it to this
/// value.
fn connected_now(&mut self, raise_to_min: u32) {
let now = Instant::now();
Addr::flush(&mut self.score, now);
self.score.connected_since = Some(now);
if self.score.score < raise_to_min {
self.score.score = raise_to_min;
}
}
/// Applies a modification to the score.
fn adjust_score(&mut self, score_diff: i32) {
Addr::flush(&mut self.score, Instant::now());
if score_diff >= 0 {
self.score.score = cmp::min(MAX_SCORE, self.score.score + score_diff as u32);
} else {
self.score.score = self.score.score.saturating_sub(-score_diff as u32);
}
}
/// Sets the addr to disconnected and applies a modification to the score.
fn disconnected_now(&mut self, score_diff: i32) {
Addr::flush(&mut self.score, Instant::now());
self.score.connected_since = None;
if score_diff >= 0 {
self.score.score = cmp::min(MAX_SCORE, self.score.score + score_diff as u32);
} else {
self.score.score = self.score.score.saturating_sub(-score_diff as u32);
}
}
/// Returns true if we are connected to this addr.
fn is_connected(&self) -> bool {
self.score.connected_since.is_some()
}
/// Returns the score, and true if we are connected to this addr.
fn score_and_is_connected(&mut self) -> (u32, bool) {
Addr::flush(&mut self.score, Instant::now());
let is_connected = self.score.connected_since.is_some();
(self.score.score, is_connected)
}
/// Updates `score` and `latest_score_update`, and returns the score.
fn score(&mut self) -> u32 {
Addr::flush(&mut self.score, Instant::now());
self.score.score
}
fn flush(score: &mut AddrScore, now: Instant) {
if let Some(connected_since) = score.connected_since {
let potential_score: u32 = div_dur_with_dur(now - connected_since, CONNEC_DURATION_PER_SCORE);
// We flush when we connect to an address.
debug_assert!(score.latest_score_update >= connected_since);
let effective_score: u32 =
div_dur_with_dur(score.latest_score_update - connected_since, CONNEC_DURATION_PER_SCORE);
let to_add = potential_score.saturating_sub(effective_score);
score.score = cmp::min(MAX_SCORE, score.score + to_add);
}
score.latest_score_update = now;
}
}
/// Divides a `Duration` with a `Duration`. This exists in the stdlib but isn't stable yet.
// TODO: remove this function once stable
fn div_dur_with_dur(a: Duration, b: Duration) -> u32 {
let a_ms = a.as_secs() * 1_000_000 + u64::from(a.subsec_micros());
let b_ms = b.as_secs() * 1_000_000 + u64::from(b.subsec_micros());
(a_ms / b_ms) as u32
}
/// Serialized version of a `PeerInfo`. Suitable for storage in the cache file.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SerializedPeerInfo {
addrs: Vec,
}
/// Serialized version of an `Addr`. Suitable for storage in the cache file.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SerializedAddr {
addr: String,
expires: SystemTime,
score: u32,
}
impl<'a> From<&'a mut Addr> for SerializedAddr {
fn from(addr: &'a mut Addr) -> SerializedAddr {
SerializedAddr {
addr: addr.addr.to_string(),
expires: addr.expires,
score: addr.score(),
}
}
}
/// Attempts to load storage from a file.
/// Ignores any entry equal to `local_peer_id`.
/// Deletes the file and returns an empty map if the file doesn't exist, cannot be opened
/// or is corrupted.
fn try_load(path: impl AsRef, local_peer_id: &PeerId) -> FnvHashMap {
let path = path.as_ref();
if !path.exists() {
debug!(target: "sub-libp2p", "Peer storage file {:?} doesn't exist", path);
return Default::default()
}
let mut file = match fs::File::open(path) {
// TODO: the capacity of the BufReader is kind of arbitrary ; decide better
Ok(f) => BufReader::with_capacity(1024 * 1024, f),
Err(err) => {
warn!(target: "sub-libp2p", "Failed to open peer storage file: {:?}", err);
info!(target: "sub-libp2p", "Deleting peer storage file {:?}", path);
let _ = fs::remove_file(path);
return Default::default()
}
};
// We want to support empty files (and treat them as an empty recordset). Unfortunately
// `serde_json` will always produce an error if we do this ("unexpected EOF at line 0
// column 0"). Therefore we start by reading one byte from the file in order to check
// for EOF.
let mut first_byte = [0];
let num_read = match file.read(&mut first_byte) {
Ok(f) => f,
Err(err) => {
// TODO: DRY
warn!(target: "sub-libp2p", "Failed to read peer storage file: {:?}", err);
info!(target: "sub-libp2p", "Deleting peer storage file {:?}", path);
let _ = fs::remove_file(path);
return Default::default()
}
};
if num_read == 0 {
// File is empty.
debug!(target: "sub-libp2p", "Peer storage file {:?} is empty", path);
Default::default()
} else {
let data = Cursor::new(first_byte).chain(file);
match serde_json::from_reader::<_, serde_json::Value>(data) {
Ok(serde_json::Value::Null) => Default::default(),
Ok(serde_json::Value::Object(map)) =>
deserialize_tolerant(map.into_iter(), local_peer_id),
Ok(_) | Err(_) => {
// The `Ok(_)` case means that the file doesn't contain a map.
let _ = fs::remove_file(path);
Default::default()
},
}
}
}
/// Attempts to turn a deserialized version of the storage into the final version.
///
/// Skips entries that are invalid or equal to `local_peer_id`.
fn deserialize_tolerant(
iter: impl Iterator
- ,
local_peer_id: &PeerId
) -> FnvHashMap {
let now = Instant::now();
let now_systime = SystemTime::now();
let mut out = FnvHashMap::default();
for (peer, info) in iter {
let peer: PeerId = match peer.parse() {
Ok(p) => p,
Err(_) => continue,
};
if &peer == local_peer_id {
continue
}
let info: SerializedPeerInfo = match serde_json::from_value(info) {
Ok(i) => i,
Err(_) => continue,
};
let mut addrs = Vec::with_capacity(info.addrs.len());
for addr in info.addrs {
let multiaddr = match addr.addr.parse() {
Ok(a) => a,
Err(_) => continue,
};
if addr.expires < now_systime {
continue
}
addrs.push(Addr {
addr: multiaddr,
expires: addr.expires,
next_back_off: FIRST_CONNECT_FAIL_BACKOFF,
back_off_until: now,
score: AddrScore {
connected_since: None,
score: addr.score,
latest_score_update: now,
},
});
}
if addrs.is_empty() {
continue
}
out.insert(peer, PeerInfo { addrs });
}
out
}
/// Attempts to turn a deserialized version of the storage into the final version.
///
/// Skips entries that are invalid or expired.
fn serialize(out: W, map: &mut FnvHashMap) -> Result<(), IoError> {
let now = SystemTime::now();
let array: FnvHashMap<_, _> = map.iter_mut().filter_map(|(peer, info)| {
if info.addrs.is_empty() {
return None
}
let peer = peer.to_base58();
let info = SerializedPeerInfo {
addrs: info.addrs.iter_mut()
.filter_map(|a| if a.expires > now || a.is_connected() {
Some(a.into())
} else {
None
})
.collect(),
};
Some((peer, info))
}).collect();
serde_json::to_writer_pretty(out, &array)
.map_err(|err| IoError::new(IoErrorKind::Other, err))
}