diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index cebb3aa5b4..d5f8498b5f 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -1662,6 +1662,14 @@ name = "linked-hash-map" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "linked_hash_set" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "lock_api" version = "0.1.5" @@ -3908,6 +3916,7 @@ dependencies = [ "fork-tree 0.1.0", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "linked_hash_set 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5124,6 +5133,7 @@ dependencies = [ "checksum libsecp256k1 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "688e8d65e495567c2c35ea0001b26b9debf0b4ea11f8cccc954233b75fc3428a" "checksum linked-hash-map 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7860ec297f7008ff7a1e3382d7f7e1dcd69efc94751a2284bafc3d013c2aa939" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" +"checksum linked_hash_set 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3c7c91c4c7bbeb4f2f7c4e5be11e6a05bd6830bc37249c47ce1ad86ad453ff9c" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" diff --git a/substrate/core/network/Cargo.toml b/substrate/core/network/Cargo.toml index c760af92db..b6d51a27d1 100644 --- a/substrate/core/network/Cargo.toml +++ b/substrate/core/network/Cargo.toml @@ -16,6 +16,7 @@ error-chain = "0.12" bitflags = "1.0" futures = "0.1.17" linked-hash-map = "0.5" +linked_hash_set = "0.1.3" lru-cache = "0.1.1" rustc-hex = "2.0" rand = "0.6" diff --git a/substrate/core/network/src/lib.rs b/substrate/core/network/src/lib.rs index 460b0822a9..31c90cb311 100644 --- a/substrate/core/network/src/lib.rs +++ b/substrate/core/network/src/lib.rs @@ -27,6 +27,7 @@ mod protocol; mod chain; mod blocks; mod on_demand; +mod util; pub mod config; pub mod consensus_gossip; pub mod error; diff --git a/substrate/core/network/src/protocol.rs b/substrate/core/network/src/protocol.rs index 5840371e32..bc30d32008 100644 --- a/substrate/core/network/src/protocol.rs +++ b/substrate/core/network/src/protocol.rs @@ -29,15 +29,13 @@ use crate::sync::{ChainSync, Status as SyncStatus, SyncState}; use crate::service::{NetworkChan, NetworkMsg, TransactionPool, ExHashT}; use crate::config::{ProtocolConfig, Roles}; use rustc_hex::ToHex; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use std::thread; -use std::time; -use std::cmp; +use std::{cmp, num::NonZeroUsize, thread, time}; use log::{trace, debug, warn}; use crate::chain::Client; use client::light::fetcher::ChangesProof; -use crate::error; +use crate::{error, util::LruHashSet}; const REQUEST_TIMEOUT_SEC: u64 = 40; const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1000); @@ -90,9 +88,9 @@ struct Peer { /// Requests we are no longer insterested in. obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. - known_extrinsics: HashSet, + known_extrinsics: LruHashSet, /// Holds a set of blocks known to this peer. - known_blocks: HashSet, + known_blocks: LruHashSet, /// Request counter, next_request_id: message::RequestId, } @@ -685,6 +683,8 @@ impl, H: ExHashT> Protocol { } } + let cache_limit = NonZeroUsize::new(1_000_000).expect("1_000_000 > 0; qed"); + let peer = Peer { info: PeerInfo { protocol_version: status.version, @@ -693,8 +693,8 @@ impl, H: ExHashT> Protocol { best_number: status.best_number }, block_request: None, - known_extrinsics: HashSet::new(), - known_blocks: HashSet::new(), + known_extrinsics: LruHashSet::new(cache_limit), + known_blocks: LruHashSet::new(cache_limit), next_request_id: 0, obsolete_requests: HashMap::new(), }; diff --git a/substrate/core/network/src/util.rs b/substrate/core/network/src/util.rs new file mode 100644 index 0000000000..5e9e64aae7 --- /dev/null +++ b/substrate/core/network/src/util.rs @@ -0,0 +1,76 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use linked_hash_set::LinkedHashSet; +use std::{hash::Hash, num::NonZeroUsize}; + +/// Wrapper around `LinkedHashSet` which grows bounded. +/// +/// In the limit, for each element inserted the oldest existing element will be removed. +#[derive(Debug, Clone)] +pub(crate) struct LruHashSet { + set: LinkedHashSet, + limit: NonZeroUsize +} + +impl LruHashSet { + /// Create a new `LruHashSet` with the given (exclusive) limit. + pub(crate) fn new(limit: NonZeroUsize) -> Self { + Self { set: LinkedHashSet::new(), limit } + } + + /// Insert element into the set. + /// + /// Returns `true` if this is a new element to the set, `false` otherwise. + /// Maintains the limit of the set by removing the oldest entry if necessary. + /// Inserting the same element will update its LRU position. + pub(crate) fn insert(&mut self, e: T) -> bool { + if self.set.insert(e) { + if self.set.len() == usize::from(self.limit) { + self.set.pop_front(); // remove oldest entry + } + return true + } + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn maintains_limit() { + let three = NonZeroUsize::new(3).unwrap(); + let mut set = LruHashSet::::new(three); + + // First element. + assert!(set.insert(1)); + assert_eq!(vec![&1], set.set.iter().collect::>()); + + // Second element. + assert!(set.insert(2)); + assert_eq!(vec![&1, &2], set.set.iter().collect::>()); + + // Inserting the same element updates its LRU position. + assert!(!set.insert(1)); + assert_eq!(vec![&2, &1], set.set.iter().collect::>()); + + // We reached the limit. The next element forces the oldest one out. + assert!(set.insert(3)); + assert_eq!(vec![&1, &3], set.set.iter().collect::>()); + } +}