mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-04-26 19:17:58 +00:00
Reorganising the repository - external renames and moves (#4074)
* Adding first rough ouline of the repository structure * Remove old CI stuff * add title * formatting fixes * move node-exits job's script to scripts dir * Move docs into subdir * move to bin * move maintainence scripts, configs and helpers into its own dir * add .local to ignore * move core->client * start up 'test' area * move test client * move test runtime * make test move compile * Add dependencies rule enforcement. * Fix indexing. * Update docs to reflect latest changes * Moving /srml->/paint * update docs * move client/sr-* -> primitives/ * clean old readme * remove old broken code in rhd * update lock * Step 1. * starting to untangle client * Fix after merge. * start splitting out client interfaces * move children and blockchain interfaces * Move trie and state-machine to primitives. * Fix WASM builds. * fixing broken imports * more interface moves * move backend and light to interfaces * move CallExecutor * move cli off client * moving around more interfaces * re-add consensus crates into the mix * fix subkey path * relieve client from executor * starting to pull out client from grandpa * move is_decendent_of out of client * grandpa still depends on client directly * lemme tests pass * rename srml->paint * Make it compile. * rename interfaces->client-api * Move keyring to primitives. * fixup libp2p dep * fix broken use * allow dependency enforcement to fail * move fork-tree * Moving wasm-builder * make env * move build-script-utils * fixup broken crate depdencies and names * fix imports for authority discovery * fix typo * update cargo.lock * fixing imports * Fix paths and add missing crates * re-add missing crates
This commit is contained in:
Benjamin Kampmann
committed by
Bastian Köcher
Bastian Köcher
parent
becc3b0a4f
commit
60e5011c72
@@ -0,0 +1,312 @@
|
||||
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use std::mem;
|
||||
use std::cmp;
|
||||
use std::ops::Range;
|
||||
use std::collections::{HashMap, BTreeMap};
|
||||
use std::collections::hash_map::Entry;
|
||||
use log::trace;
|
||||
use libp2p::PeerId;
|
||||
use sr_primitives::traits::{Block as BlockT, NumberFor, One};
|
||||
use crate::message;
|
||||
|
||||
/// Block data with origin.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct BlockData<B: BlockT> {
|
||||
/// The Block Message from the wire
|
||||
pub block: message::BlockData<B>,
|
||||
/// The peer, we received this from
|
||||
pub origin: Option<PeerId>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum BlockRangeState<B: BlockT> {
|
||||
Downloading {
|
||||
len: NumberFor<B>,
|
||||
downloading: u32,
|
||||
},
|
||||
Complete(Vec<BlockData<B>>),
|
||||
}
|
||||
|
||||
impl<B: BlockT> BlockRangeState<B> {
|
||||
pub fn len(&self) -> NumberFor<B> {
|
||||
match *self {
|
||||
BlockRangeState::Downloading { len, .. } => len,
|
||||
BlockRangeState::Complete(ref blocks) => (blocks.len() as u32).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A collection of blocks being downloaded.
|
||||
#[derive(Default)]
|
||||
pub struct BlockCollection<B: BlockT> {
|
||||
/// Downloaded blocks.
|
||||
blocks: BTreeMap<NumberFor<B>, BlockRangeState<B>>,
|
||||
peer_requests: HashMap<PeerId, NumberFor<B>>,
|
||||
}
|
||||
|
||||
impl<B: BlockT> BlockCollection<B> {
|
||||
/// Create a new instance.
|
||||
pub fn new() -> Self {
|
||||
BlockCollection {
|
||||
blocks: BTreeMap::new(),
|
||||
peer_requests: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear everything.
|
||||
pub fn clear(&mut self) {
|
||||
self.blocks.clear();
|
||||
self.peer_requests.clear();
|
||||
}
|
||||
|
||||
/// Insert a set of blocks into collection.
|
||||
pub fn insert(&mut self, start: NumberFor<B>, blocks: Vec<message::BlockData<B>>, who: PeerId) {
|
||||
if blocks.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
match self.blocks.get(&start) {
|
||||
Some(&BlockRangeState::Downloading { .. }) => {
|
||||
trace!(target: "sync", "Inserting block data still marked as being downloaded: {}", start);
|
||||
},
|
||||
Some(&BlockRangeState::Complete(ref existing)) if existing.len() >= blocks.len() => {
|
||||
trace!(target: "sync", "Ignored block data already downloaded: {}", start);
|
||||
return;
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
|
||||
self.blocks.insert(start, BlockRangeState::Complete(blocks.into_iter()
|
||||
.map(|b| BlockData { origin: Some(who.clone()), block: b }).collect()));
|
||||
}
|
||||
|
||||
/// Returns a set of block hashes that require a header download. The returned set is marked as being downloaded.
|
||||
pub fn needed_blocks(
|
||||
&mut self,
|
||||
who: PeerId,
|
||||
count: usize,
|
||||
peer_best: NumberFor<B>,
|
||||
common: NumberFor<B>,
|
||||
max_parallel: u32,
|
||||
max_ahead: u32,
|
||||
) -> Option<Range<NumberFor<B>>>
|
||||
{
|
||||
if peer_best <= common {
|
||||
// Bail out early
|
||||
return None;
|
||||
}
|
||||
// First block number that we need to download
|
||||
let first_different = common + <NumberFor<B>>::one();
|
||||
let count = (count as u32).into();
|
||||
let (mut range, downloading) = {
|
||||
let mut downloading_iter = self.blocks.iter().peekable();
|
||||
let mut prev: Option<(&NumberFor<B>, &BlockRangeState<B>)> = None;
|
||||
loop {
|
||||
let next = downloading_iter.next();
|
||||
break match &(prev, next) {
|
||||
&(Some((start, &BlockRangeState::Downloading { ref len, downloading })), _)
|
||||
if downloading < max_parallel =>
|
||||
(*start .. *start + *len, downloading),
|
||||
&(Some((start, r)), Some((next_start, _))) if *start + r.len() < *next_start =>
|
||||
(*start + r.len() .. cmp::min(*next_start, *start + r.len() + count), 0), // gap
|
||||
&(Some((start, r)), None) =>
|
||||
(*start + r.len() .. *start + r.len() + count, 0), // last range
|
||||
&(None, None) =>
|
||||
(first_different .. first_different + count, 0), // empty
|
||||
&(None, Some((start, _))) if *start > first_different =>
|
||||
(first_different .. cmp::min(first_different + count, *start), 0), // gap at the start
|
||||
_ => {
|
||||
prev = next;
|
||||
continue
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
// crop to peers best
|
||||
if range.start > peer_best {
|
||||
trace!(target: "sync", "Out of range for peer {} ({} vs {})", who, range.start, peer_best);
|
||||
return None;
|
||||
}
|
||||
range.end = cmp::min(peer_best + One::one(), range.end);
|
||||
|
||||
if self.blocks.iter().next().map_or(false, |(n, _)| range.start > *n + max_ahead.into()) {
|
||||
trace!(target: "sync", "Too far ahead for peer {} ({})", who, range.start);
|
||||
return None;
|
||||
}
|
||||
|
||||
self.peer_requests.insert(who, range.start);
|
||||
self.blocks.insert(range.start, BlockRangeState::Downloading {
|
||||
len: range.end - range.start,
|
||||
downloading: downloading + 1
|
||||
});
|
||||
if range.end <= range.start {
|
||||
panic!("Empty range {:?}, count={}, peer_best={}, common={}, blocks={:?}",
|
||||
range, count, peer_best, common, self.blocks);
|
||||
}
|
||||
Some(range)
|
||||
}
|
||||
|
||||
/// Get a valid chain of blocks ordered in descending order and ready for importing into blockchain.
|
||||
pub fn drain(&mut self, from: NumberFor<B>) -> Vec<BlockData<B>> {
|
||||
let mut drained = Vec::new();
|
||||
let mut ranges = Vec::new();
|
||||
{
|
||||
let mut prev = from;
|
||||
for (start, range_data) in &mut self.blocks {
|
||||
match range_data {
|
||||
&mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => {
|
||||
prev = *start + (blocks.len() as u32).into();
|
||||
let mut blocks = mem::replace(blocks, Vec::new());
|
||||
drained.append(&mut blocks);
|
||||
ranges.push(*start);
|
||||
},
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
for r in ranges {
|
||||
self.blocks.remove(&r);
|
||||
}
|
||||
trace!(target: "sync", "Drained {} blocks", drained.len());
|
||||
drained
|
||||
}
|
||||
|
||||
pub fn clear_peer_download(&mut self, who: &PeerId) {
|
||||
match self.peer_requests.entry(who.clone()) {
|
||||
Entry::Occupied(entry) => {
|
||||
let start = entry.remove();
|
||||
let remove = match self.blocks.get_mut(&start) {
|
||||
Some(&mut BlockRangeState::Downloading { ref mut downloading, .. }) if *downloading > 1 => {
|
||||
*downloading = *downloading - 1;
|
||||
false
|
||||
},
|
||||
Some(&mut BlockRangeState::Downloading { .. }) => {
|
||||
true
|
||||
},
|
||||
_ => {
|
||||
false
|
||||
}
|
||||
};
|
||||
if remove {
|
||||
self.blocks.remove(&start);
|
||||
}
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::{BlockCollection, BlockData, BlockRangeState};
|
||||
use crate::{message, PeerId};
|
||||
use sr_primitives::testing::{Block as RawBlock, ExtrinsicWrapper};
|
||||
use primitives::H256;
|
||||
|
||||
type Block = RawBlock<ExtrinsicWrapper<u64>>;
|
||||
|
||||
fn is_empty(bc: &BlockCollection<Block>) -> bool {
|
||||
bc.blocks.is_empty() &&
|
||||
bc.peer_requests.is_empty()
|
||||
}
|
||||
|
||||
fn generate_blocks(n: usize) -> Vec<message::BlockData<Block>> {
|
||||
(0 .. n).map(|_| message::generic::BlockData {
|
||||
hash: H256::random(),
|
||||
header: None,
|
||||
body: None,
|
||||
message_queue: None,
|
||||
receipt: None,
|
||||
justification: None,
|
||||
}).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_clear() {
|
||||
let mut bc = BlockCollection::new();
|
||||
assert!(is_empty(&bc));
|
||||
bc.insert(1, generate_blocks(100), PeerId::random());
|
||||
assert!(!is_empty(&bc));
|
||||
bc.clear();
|
||||
assert!(is_empty(&bc));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insert_blocks() {
|
||||
let mut bc = BlockCollection::new();
|
||||
assert!(is_empty(&bc));
|
||||
let peer0 = PeerId::random();
|
||||
let peer1 = PeerId::random();
|
||||
let peer2 = PeerId::random();
|
||||
|
||||
let blocks = generate_blocks(150);
|
||||
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(1 .. 41));
|
||||
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(41 .. 81));
|
||||
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 0, 1, 200), Some(81 .. 121));
|
||||
|
||||
bc.clear_peer_download(&peer1);
|
||||
bc.insert(41, blocks[41..81].to_vec(), peer1.clone());
|
||||
assert_eq!(bc.drain(1), vec![]);
|
||||
assert_eq!(bc.needed_blocks(peer1.clone(), 40, 150, 0, 1, 200), Some(121 .. 151));
|
||||
bc.clear_peer_download(&peer0);
|
||||
bc.insert(1, blocks[1..11].to_vec(), peer0.clone());
|
||||
|
||||
assert_eq!(bc.needed_blocks(peer0.clone(), 40, 150, 0, 1, 200), Some(11 .. 41));
|
||||
assert_eq!(bc.drain(1), blocks[1..11].iter()
|
||||
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>());
|
||||
|
||||
bc.clear_peer_download(&peer0);
|
||||
bc.insert(11, blocks[11..41].to_vec(), peer0.clone());
|
||||
|
||||
let drained = bc.drain(12);
|
||||
assert_eq!(drained[..30], blocks[11..41].iter()
|
||||
.map(|b| BlockData { block: b.clone(), origin: Some(peer0.clone()) }).collect::<Vec<_>>()[..]);
|
||||
assert_eq!(drained[30..], blocks[41..81].iter()
|
||||
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
|
||||
|
||||
bc.clear_peer_download(&peer2);
|
||||
assert_eq!(bc.needed_blocks(peer2.clone(), 40, 150, 80, 1, 200), Some(81 .. 121));
|
||||
bc.clear_peer_download(&peer2);
|
||||
bc.insert(81, blocks[81..121].to_vec(), peer2.clone());
|
||||
bc.clear_peer_download(&peer1);
|
||||
bc.insert(121, blocks[121..150].to_vec(), peer1.clone());
|
||||
|
||||
assert_eq!(bc.drain(80), vec![]);
|
||||
let drained = bc.drain(81);
|
||||
assert_eq!(drained[..40], blocks[81..121].iter()
|
||||
.map(|b| BlockData { block: b.clone(), origin: Some(peer2.clone()) }).collect::<Vec<_>>()[..]);
|
||||
assert_eq!(drained[40..], blocks[121..150].iter()
|
||||
.map(|b| BlockData { block: b.clone(), origin: Some(peer1.clone()) }).collect::<Vec<_>>()[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_gap() {
|
||||
let mut bc: BlockCollection<Block> = BlockCollection::new();
|
||||
bc.blocks.insert(100, BlockRangeState::Downloading {
|
||||
len: 128,
|
||||
downloading: 1,
|
||||
});
|
||||
let blocks = generate_blocks(10).into_iter().map(|b| BlockData { block: b, origin: None }).collect();
|
||||
bc.blocks.insert(114305, BlockRangeState::Complete(blocks));
|
||||
|
||||
let peer0 = PeerId::random();
|
||||
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 000, 1, 200), Some(1 .. 100));
|
||||
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200), None); // too far ahead
|
||||
assert_eq!(bc.needed_blocks(peer0.clone(), 128, 10000, 600, 1, 200000), Some(100 + 128 .. 100 + 128 + 128));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
// Copyright 2017-2018 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
use client_api::error::Error as ClientError;
|
||||
use crate::protocol::sync::{PeerSync, PeerSyncState};
|
||||
use fork_tree::ForkTree;
|
||||
use libp2p::PeerId;
|
||||
use log::{debug, warn};
|
||||
use sr_primitives::traits::{Block as BlockT, NumberFor, Zero};
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
// Time to wait before trying to get the same extra data from the same peer.
|
||||
const EXTRA_RETRY_WAIT: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Pending extra data request for the given block (hash and number).
|
||||
pub(crate) type ExtraRequest<B> = (<B as BlockT>::Hash, NumberFor<B>);
|
||||
|
||||
/// Manages pending block extra data (e.g. justification) requests.
|
||||
///
|
||||
/// Multiple extras may be requested for competing forks, or for the same branch
|
||||
/// at different (increasing) heights. This structure will guarantee that extras
|
||||
/// are fetched in-order, and that obsolete changes are pruned (when finalizing a
|
||||
/// competing fork).
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ExtraRequests<B: BlockT> {
|
||||
tree: ForkTree<B::Hash, NumberFor<B>, ()>,
|
||||
/// best finalized block number that we have seen since restart
|
||||
best_seen_finalized_number: NumberFor<B>,
|
||||
/// requests which have been queued for later processing
|
||||
pending_requests: VecDeque<ExtraRequest<B>>,
|
||||
/// requests which are currently underway to some peer
|
||||
active_requests: HashMap<PeerId, ExtraRequest<B>>,
|
||||
/// previous requests without response
|
||||
failed_requests: HashMap<ExtraRequest<B>, Vec<(PeerId, Instant)>>,
|
||||
/// successful requests
|
||||
importing_requests: HashSet<ExtraRequest<B>>,
|
||||
}
|
||||
|
||||
impl<B: BlockT> ExtraRequests<B> {
|
||||
pub(crate) fn new() -> Self {
|
||||
ExtraRequests {
|
||||
tree: ForkTree::new(),
|
||||
best_seen_finalized_number: Zero::zero(),
|
||||
pending_requests: VecDeque::new(),
|
||||
active_requests: HashMap::new(),
|
||||
failed_requests: HashMap::new(),
|
||||
importing_requests: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset all state as if returned from `new`.
|
||||
pub(crate) fn reset(&mut self) {
|
||||
self.tree = ForkTree::new();
|
||||
self.pending_requests.clear();
|
||||
self.active_requests.clear();
|
||||
self.failed_requests.clear();
|
||||
}
|
||||
|
||||
/// Returns an iterator-like struct that yields peers which extra
|
||||
/// requests can be sent to.
|
||||
pub(crate) fn matcher(&mut self) -> Matcher<B> {
|
||||
Matcher::new(self)
|
||||
}
|
||||
|
||||
/// Queue an extra data request to be considered by the `Matcher`.
|
||||
pub(crate) fn schedule<F>(&mut self, request: ExtraRequest<B>, is_descendent_of: F)
|
||||
where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
|
||||
{
|
||||
match self.tree.import(request.0, request.1, (), &is_descendent_of) {
|
||||
Ok(true) => {
|
||||
// this is a new root so we add it to the current `pending_requests`
|
||||
self.pending_requests.push_back((request.0, request.1));
|
||||
}
|
||||
Err(fork_tree::Error::Revert) => {
|
||||
// we have finalized further than the given request, presumably
|
||||
// by some other part of the system (not sync). we can safely
|
||||
// ignore the `Revert` error.
|
||||
return;
|
||||
},
|
||||
Err(err) => {
|
||||
debug!(target: "sync", "Failed to insert request {:?} into tree: {:?}", request, err);
|
||||
return;
|
||||
}
|
||||
_ => ()
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry any pending request if a peer disconnected.
|
||||
pub(crate) fn peer_disconnected(&mut self, who: &PeerId) {
|
||||
if let Some(request) = self.active_requests.remove(who) {
|
||||
self.pending_requests.push_front(request);
|
||||
}
|
||||
}
|
||||
|
||||
/// Processes the response for the request previously sent to the given peer.
|
||||
pub(crate) fn on_response<R>(&mut self, who: PeerId, resp: Option<R>) -> Option<(PeerId, B::Hash, NumberFor<B>, R)> {
|
||||
// we assume that the request maps to the given response, this is
|
||||
// currently enforced by the outer network protocol before passing on
|
||||
// messages to chain sync.
|
||||
if let Some(request) = self.active_requests.remove(&who) {
|
||||
if let Some(r) = resp {
|
||||
self.importing_requests.insert(request);
|
||||
return Some((who, request.0, request.1, r))
|
||||
}
|
||||
self.failed_requests.entry(request).or_insert(Vec::new()).push((who, Instant::now()));
|
||||
self.pending_requests.push_front(request);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Removes any pending extra requests for blocks lower than the given best finalized.
|
||||
pub(crate) fn on_block_finalized<F>(
|
||||
&mut self,
|
||||
best_finalized_hash: &B::Hash,
|
||||
best_finalized_number: NumberFor<B>,
|
||||
is_descendent_of: F
|
||||
) -> Result<(), fork_tree::Error<ClientError>>
|
||||
where F: Fn(&B::Hash, &B::Hash) -> Result<bool, ClientError>
|
||||
{
|
||||
let request = (*best_finalized_hash, best_finalized_number);
|
||||
|
||||
if self.try_finalize_root::<()>(request, Ok(request), false) {
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
if best_finalized_number > self.best_seen_finalized_number {
|
||||
// normally we'll receive finality notifications for every block => finalize would be enough
|
||||
// but if many blocks are finalized at once, some notifications may be omitted
|
||||
// => let's use finalize_with_ancestors here
|
||||
match self.tree.finalize_with_ancestors(
|
||||
best_finalized_hash,
|
||||
best_finalized_number,
|
||||
&is_descendent_of,
|
||||
) {
|
||||
Err(fork_tree::Error::Revert) => {
|
||||
// we might have finalized further already in which case we
|
||||
// will get a `Revert` error which we can safely ignore.
|
||||
},
|
||||
Err(err) => return Err(err),
|
||||
Ok(_) => {},
|
||||
}
|
||||
|
||||
self.best_seen_finalized_number = best_finalized_number;
|
||||
}
|
||||
|
||||
let roots = self.tree.roots().collect::<HashSet<_>>();
|
||||
|
||||
self.pending_requests.retain(|(h, n)| roots.contains(&(h, n, &())));
|
||||
self.active_requests.retain(|_, (h, n)| roots.contains(&(h, n, &())));
|
||||
self.failed_requests.retain(|(h, n), _| roots.contains(&(h, n, &())));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Try to finalize pending root.
|
||||
///
|
||||
/// Returns true if import of this request has been scheduled.
|
||||
pub(crate) fn try_finalize_root<E>(
|
||||
&mut self,
|
||||
request: ExtraRequest<B>,
|
||||
result: Result<ExtraRequest<B>, E>,
|
||||
reschedule_on_failure: bool
|
||||
) -> bool
|
||||
{
|
||||
if !self.importing_requests.remove(&request) {
|
||||
return false
|
||||
}
|
||||
|
||||
let (finalized_hash, finalized_number) = match result {
|
||||
Ok(req) => (req.0, req.1),
|
||||
Err(_) => {
|
||||
if reschedule_on_failure {
|
||||
self.pending_requests.push_front(request);
|
||||
}
|
||||
return true
|
||||
}
|
||||
};
|
||||
|
||||
if self.tree.finalize_root(&finalized_hash).is_none() {
|
||||
warn!(target: "sync", "Imported {:?} {:?} which isn't a root in the tree: {:?}",
|
||||
finalized_hash,
|
||||
finalized_number,
|
||||
self.tree.roots().collect::<Vec<_>>()
|
||||
);
|
||||
return true
|
||||
}
|
||||
|
||||
self.failed_requests.clear();
|
||||
self.active_requests.clear();
|
||||
self.pending_requests.clear();
|
||||
self.pending_requests.extend(self.tree.roots().map(|(&h, &n, _)| (h, n)));
|
||||
self.best_seen_finalized_number = finalized_number;
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Matches peers with pending extra requests.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Matcher<'a, B: BlockT> {
|
||||
/// Length of pending requests collection.
|
||||
/// Used to ensure we do not loop more than once over all pending requests.
|
||||
remaining: usize,
|
||||
extras: &'a mut ExtraRequests<B>
|
||||
}
|
||||
|
||||
impl<'a, B: BlockT> Matcher<'a, B> {
|
||||
fn new(extras: &'a mut ExtraRequests<B>) -> Self {
|
||||
Matcher {
|
||||
remaining: extras.pending_requests.len(),
|
||||
extras
|
||||
}
|
||||
}
|
||||
|
||||
/// Finds a peer to which a pending request can be sent.
|
||||
///
|
||||
/// Peers are filtered according to the current known best block (i.e. we won't
|
||||
/// send an extra request for block #10 to a peer at block #2), and we also
|
||||
/// throttle requests to the same peer if a previous request yielded no results.
|
||||
///
|
||||
/// This method returns as soon as it finds a peer that should be able to answer
|
||||
/// our request. If no request is pending or no peer can handle it, `None` is
|
||||
/// returned instead.
|
||||
///
|
||||
/// # Note
|
||||
///
|
||||
/// The returned `PeerId` (if any) is guaranteed to come from the given `peers`
|
||||
/// argument.
|
||||
pub(crate) fn next(&mut self, peers: &HashMap<PeerId, PeerSync<B>>) -> Option<(PeerId, ExtraRequest<B>)> {
|
||||
if self.remaining == 0 {
|
||||
return None
|
||||
}
|
||||
|
||||
// clean up previously failed requests so we can retry again
|
||||
for requests in self.extras.failed_requests.values_mut() {
|
||||
requests.retain(|(_, instant)| instant.elapsed() < EXTRA_RETRY_WAIT);
|
||||
}
|
||||
|
||||
while let Some(request) = self.extras.pending_requests.pop_front() {
|
||||
for (peer, sync) in peers.iter().filter(|(_, sync)| sync.state == PeerSyncState::Available) {
|
||||
// only ask peers that have synced at least up to the block number that we're asking the extra for
|
||||
if sync.best_number < request.1 {
|
||||
continue
|
||||
}
|
||||
// don't request to any peers that already have pending requests
|
||||
if self.extras.active_requests.contains_key(peer) {
|
||||
continue
|
||||
}
|
||||
// only ask if the same request has not failed for this peer before
|
||||
if self.extras.failed_requests.get(&request).map(|rr| rr.iter().any(|i| &i.0 == peer)).unwrap_or(false) {
|
||||
continue
|
||||
}
|
||||
self.extras.active_requests.insert(peer.clone(), request);
|
||||
return Some((peer.clone(), request))
|
||||
}
|
||||
|
||||
self.extras.pending_requests.push_back(request);
|
||||
self.remaining -= 1;
|
||||
|
||||
if self.remaining == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::protocol::sync::PeerSync;
|
||||
use client_api::error::Error as ClientError;
|
||||
use quickcheck::{Arbitrary, Gen, QuickCheck, StdThreadGen};
|
||||
use rand::Rng;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use super::*;
|
||||
use test_client::runtime::{Block, BlockNumber, Hash};
|
||||
|
||||
#[test]
|
||||
fn requests_are_processed_in_order() {
|
||||
fn property(mut peers: ArbitraryPeers) {
|
||||
let mut requests = ExtraRequests::<Block>::new();
|
||||
|
||||
let num_peers_available = peers.0.values()
|
||||
.filter(|s| s.state == PeerSyncState::Available).count();
|
||||
|
||||
for i in 0 .. num_peers_available {
|
||||
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
|
||||
}
|
||||
|
||||
let pending = requests.pending_requests.clone();
|
||||
let mut m = requests.matcher();
|
||||
|
||||
for p in &pending {
|
||||
let (peer, r) = m.next(&peers.0).unwrap();
|
||||
assert_eq!(p, &r);
|
||||
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
|
||||
}
|
||||
}
|
||||
|
||||
QuickCheck::with_gen(StdThreadGen::new(19))
|
||||
.quickcheck(property as fn(ArbitraryPeers))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_roots_schedule_new_request() {
|
||||
fn property(data: Vec<BlockNumber>) {
|
||||
let mut requests = ExtraRequests::<Block>::new();
|
||||
for (i, number) in data.into_iter().enumerate() {
|
||||
let hash = [i as u8; 32].into();
|
||||
let pending = requests.pending_requests.len();
|
||||
let is_root = requests.tree.roots().any(|(&h, &n, _)| hash == h && number == n);
|
||||
requests.schedule((hash, number), |a, b| Ok(a[0] >= b[0]));
|
||||
if !is_root {
|
||||
assert_eq!(1 + pending, requests.pending_requests.len())
|
||||
}
|
||||
}
|
||||
}
|
||||
QuickCheck::new().quickcheck(property as fn(Vec<BlockNumber>))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disconnecting_implies_rescheduling() {
|
||||
fn property(mut peers: ArbitraryPeers) -> bool {
|
||||
let mut requests = ExtraRequests::<Block>::new();
|
||||
|
||||
let num_peers_available = peers.0.values()
|
||||
.filter(|s| s.state == PeerSyncState::Available).count();
|
||||
|
||||
for i in 0 .. num_peers_available {
|
||||
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
|
||||
}
|
||||
|
||||
let mut m = requests.matcher();
|
||||
while let Some((peer, r)) = m.next(&peers.0) {
|
||||
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
|
||||
}
|
||||
|
||||
assert!(requests.pending_requests.is_empty());
|
||||
|
||||
let active_peers = requests.active_requests.keys().cloned().collect::<Vec<_>>();
|
||||
let previously_active = requests.active_requests.values().cloned().collect::<HashSet<_>>();
|
||||
|
||||
for peer in &active_peers {
|
||||
requests.peer_disconnected(peer)
|
||||
}
|
||||
|
||||
assert!(requests.active_requests.is_empty());
|
||||
|
||||
previously_active == requests.pending_requests.iter().cloned().collect::<HashSet<_>>()
|
||||
}
|
||||
|
||||
QuickCheck::with_gen(StdThreadGen::new(19))
|
||||
.quickcheck(property as fn(ArbitraryPeers) -> bool)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_response_reschedules() {
|
||||
fn property(mut peers: ArbitraryPeers) {
|
||||
let mut requests = ExtraRequests::<Block>::new();
|
||||
|
||||
let num_peers_available = peers.0.values()
|
||||
.filter(|s| s.state == PeerSyncState::Available).count();
|
||||
|
||||
for i in 0 .. num_peers_available {
|
||||
requests.schedule((Hash::random(), i as u64), |a, b| Ok(a[0] >= b[0]))
|
||||
}
|
||||
|
||||
let mut m = requests.matcher();
|
||||
while let Some((peer, r)) = m.next(&peers.0) {
|
||||
peers.0.get_mut(&peer).unwrap().state = PeerSyncState::DownloadingJustification(r.0);
|
||||
}
|
||||
|
||||
let active = requests.active_requests.iter().map(|(p, &r)| (p.clone(), r)).collect::<Vec<_>>();
|
||||
|
||||
for (peer, req) in &active {
|
||||
assert!(requests.failed_requests.get(req).is_none());
|
||||
assert!(!requests.pending_requests.contains(req));
|
||||
assert!(requests.on_response::<()>(peer.clone(), None).is_none());
|
||||
assert!(requests.pending_requests.contains(req));
|
||||
assert_eq!(1, requests.failed_requests.get(req).unwrap().iter().filter(|(p, _)| p == peer).count())
|
||||
}
|
||||
}
|
||||
|
||||
QuickCheck::with_gen(StdThreadGen::new(19))
|
||||
.quickcheck(property as fn(ArbitraryPeers))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_is_rescheduled_when_earlier_block_is_finalized() {
|
||||
let _ = ::env_logger::try_init();
|
||||
|
||||
let mut finality_proofs = ExtraRequests::<Block>::new();
|
||||
|
||||
let hash4 = [4; 32].into();
|
||||
let hash5 = [5; 32].into();
|
||||
let hash6 = [6; 32].into();
|
||||
let hash7 = [7; 32].into();
|
||||
|
||||
fn is_descendent_of(base: &Hash, target: &Hash) -> Result<bool, ClientError> {
|
||||
Ok(target[0] >= base[0])
|
||||
}
|
||||
|
||||
// make #4 last finalized block
|
||||
finality_proofs.tree.import(hash4, 4, (), &is_descendent_of).unwrap();
|
||||
finality_proofs.tree.finalize_root(&hash4);
|
||||
|
||||
// schedule request for #6
|
||||
finality_proofs.schedule((hash6, 6), is_descendent_of);
|
||||
|
||||
// receive finality proof for #5
|
||||
finality_proofs.importing_requests.insert((hash6, 6));
|
||||
finality_proofs.on_block_finalized(&hash5, 5, is_descendent_of).unwrap();
|
||||
finality_proofs.try_finalize_root::<()>((hash6, 6), Ok((hash5, 5)), true);
|
||||
|
||||
// ensure that request for #6 is still pending
|
||||
assert_eq!(finality_proofs.pending_requests.iter().collect::<Vec<_>>(), vec![&(hash6, 6)]);
|
||||
|
||||
// receive finality proof for #7
|
||||
finality_proofs.importing_requests.insert((hash6, 6));
|
||||
finality_proofs.on_block_finalized(&hash6, 6, is_descendent_of).unwrap();
|
||||
finality_proofs.on_block_finalized(&hash7, 7, is_descendent_of).unwrap();
|
||||
finality_proofs.try_finalize_root::<()>((hash6, 6), Ok((hash7, 7)), true);
|
||||
|
||||
// ensure that there's no request for #6
|
||||
assert_eq!(finality_proofs.pending_requests.iter().collect::<Vec<_>>(), Vec::<&(Hash, u64)>::new());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anecstor_roots_are_finalized_when_finality_notification_is_missed() {
|
||||
let mut finality_proofs = ExtraRequests::<Block>::new();
|
||||
|
||||
let hash4 = [4; 32].into();
|
||||
let hash5 = [5; 32].into();
|
||||
|
||||
fn is_descendent_of(base: &Hash, target: &Hash) -> Result<bool, ClientError> {
|
||||
Ok(target[0] >= base[0])
|
||||
}
|
||||
|
||||
// schedule request for #4
|
||||
finality_proofs.schedule((hash4, 4), is_descendent_of);
|
||||
|
||||
// receive finality notification for #5 (missing notification for #4!!!)
|
||||
finality_proofs.importing_requests.insert((hash4, 5));
|
||||
finality_proofs.on_block_finalized(&hash5, 5, is_descendent_of).unwrap();
|
||||
assert_eq!(finality_proofs.tree.roots().count(), 0);
|
||||
}
|
||||
|
||||
// Some Arbitrary instances to allow easy construction of random peer sets:
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ArbitraryPeerSyncState(PeerSyncState<Block>);
|
||||
|
||||
impl Arbitrary for ArbitraryPeerSyncState {
|
||||
fn arbitrary<G: Gen>(g: &mut G) -> Self {
|
||||
let s = match g.gen::<u8>() % 5 {
|
||||
0 => PeerSyncState::Available,
|
||||
// TODO: 1 => PeerSyncState::AncestorSearch(g.gen(), AncestorSearchState<B>),
|
||||
1 => PeerSyncState::DownloadingNew(g.gen::<BlockNumber>()),
|
||||
2 => PeerSyncState::DownloadingStale(Hash::random()),
|
||||
3 => PeerSyncState::DownloadingJustification(Hash::random()),
|
||||
_ => PeerSyncState::DownloadingFinalityProof(Hash::random())
|
||||
};
|
||||
ArbitraryPeerSyncState(s)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ArbitraryPeerSync(PeerSync<Block>);
|
||||
|
||||
impl Arbitrary for ArbitraryPeerSync {
|
||||
fn arbitrary<G: Gen>(g: &mut G) -> Self {
|
||||
let ps = PeerSync {
|
||||
common_number: g.gen(),
|
||||
best_hash: Hash::random(),
|
||||
best_number: g.gen(),
|
||||
state: ArbitraryPeerSyncState::arbitrary(g).0,
|
||||
recently_announced: Default::default()
|
||||
};
|
||||
ArbitraryPeerSync(ps)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ArbitraryPeers(HashMap<PeerId, PeerSync<Block>>);
|
||||
|
||||
impl Arbitrary for ArbitraryPeers {
|
||||
fn arbitrary<G: Gen>(g: &mut G) -> Self {
|
||||
let mut peers = HashMap::with_capacity(g.size());
|
||||
for _ in 0 .. g.size() {
|
||||
peers.insert(PeerId::random(), ArbitraryPeerSync::arbitrary(g).0);
|
||||
}
|
||||
ArbitraryPeers(peers)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user