feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/node/network/dispute-distribution/src/receiver/batches/batch.rs

@@ -0,0 +1,209 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, time::Instant};
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{
+	request_response::{incoming::OutgoingResponseSender, v1::DisputeRequest},
+	PeerId,
+};
+use pezkuwi_node_primitives::SignedDisputeStatement;
+use pezkuwi_primitives::{CandidateReceiptV2 as CandidateReceipt, ValidatorIndex};
+
+use crate::receiver::{BATCH_COLLECTING_INTERVAL, MIN_KEEP_BATCH_ALIVE_VOTES};
+
+use super::MAX_BATCH_LIFETIME;
+
+/// A batch of votes to be imported into the `dispute-coordinator`.
+///
+/// Vote imports are way more efficient when performed in batches, hence we batch together incoming
+/// votes until the rate of incoming votes falls below a threshold, then we import into the dispute
+/// coordinator.
+///
+/// A `Batch` keeps track of the votes to be imported and the current incoming rate, on rate update
+/// it will "flush" in case the incoming rate dropped too low, preparing the import.
+pub struct Batch {
+	/// The actual candidate this batch is concerned with.
+	candidate_receipt: CandidateReceipt,
+
+	/// Cache of `CandidateHash` (candidate_receipt.hash()).
+	candidate_hash: CandidateHash,
+
+	/// All valid votes received in this batch so far.
+	///
+	/// We differentiate between valid and invalid votes, so we can detect (and drop) duplicates,
+	/// while still allowing validators to equivocate.
+	///
+	/// Detecting and rejecting duplicates is crucial in order to effectively enforce
+	/// `MIN_KEEP_BATCH_ALIVE_VOTES` per `BATCH_COLLECTING_INTERVAL`. If we would count duplicates
+	/// here, the mechanism would be broken.
+	valid_votes: HashMap<ValidatorIndex, SignedDisputeStatement>,
+
+	/// All invalid votes received in this batch so far.
+	invalid_votes: HashMap<ValidatorIndex, SignedDisputeStatement>,
+
+	/// How many votes have been batched since the last tick/creation.
+	votes_batched_since_last_tick: u32,
+
+	/// Expiry time for the batch.
+	///
+	/// By this time the latest this batch will get flushed.
+	best_before: Instant,
+
+	/// Requesters waiting for a response.
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+}
+
+/// Result of checking a batch every `BATCH_COLLECTING_INTERVAL`.
+pub(super) enum TickResult {
+	/// Batch is still alive, please call `tick` again at the given `Instant`.
+	Alive(Batch, Instant),
+	/// Batch is done, ready for import!
+	Done(PreparedImport),
+}
+
+/// Ready for import.
+pub struct PreparedImport {
+	pub candidate_receipt: CandidateReceipt,
+	pub statements: Vec<(SignedDisputeStatement, ValidatorIndex)>,
+	/// Information about original requesters.
+	pub requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+}
+
+impl From<Batch> for PreparedImport {
+	fn from(batch: Batch) -> Self {
+		let Batch {
+			candidate_receipt,
+			valid_votes,
+			invalid_votes,
+			requesters: pending_responses,
+			..
+		} = batch;
+
+		let statements = valid_votes
+			.into_iter()
+			.chain(invalid_votes.into_iter())
+			.map(|(index, statement)| (statement, index))
+			.collect();
+
+		Self { candidate_receipt, statements, requesters: pending_responses }
+	}
+}
+
+impl Batch {
+	/// Create a new empty batch based on the given `CandidateReceipt`.
+	///
+	/// To create a `Batch` use Batches::find_batch`.
+	///
+	/// Arguments:
+	///
+	/// * `candidate_receipt` - The candidate this batch is meant to track votes for.
+	/// * `now` - current time stamp for calculating the first tick.
+	///
+	/// Returns: A batch and the first `Instant` you are supposed to call `tick`.
+	pub(super) fn new(candidate_receipt: CandidateReceipt, now: Instant) -> (Self, Instant) {
+		let s = Self {
+			candidate_hash: candidate_receipt.hash(),
+			candidate_receipt,
+			valid_votes: HashMap::new(),
+			invalid_votes: HashMap::new(),
+			votes_batched_since_last_tick: 0,
+			best_before: Instant::now() + MAX_BATCH_LIFETIME,
+			requesters: Vec::new(),
+		};
+		let next_tick = s.calculate_next_tick(now);
+		(s, next_tick)
+	}
+
+	/// Receipt of the candidate this batch is batching votes for.
+	pub fn candidate_receipt(&self) -> &CandidateReceipt {
+		&self.candidate_receipt
+	}
+
+	/// Add votes from a validator into the batch.
+	///
+	/// The statements are supposed to be the valid and invalid statements received in a
+	/// `DisputeRequest`.
+	///
+	/// The given `pending_response` is the corresponding response sender for responding to `peer`.
+	/// If at least one of the votes is new as far as this batch is concerned we record the
+	/// pending_response, for later use. In case both votes are known already, we return the
+	/// response sender as an `Err` value.
+	pub fn add_votes(
+		&mut self,
+		valid_vote: (SignedDisputeStatement, ValidatorIndex),
+		invalid_vote: (SignedDisputeStatement, ValidatorIndex),
+		peer: PeerId,
+		pending_response: OutgoingResponseSender<DisputeRequest>,
+	) -> Result<(), OutgoingResponseSender<DisputeRequest>> {
+		debug_assert!(valid_vote.0.candidate_hash() == invalid_vote.0.candidate_hash());
+		debug_assert!(valid_vote.0.candidate_hash() == &self.candidate_hash);
+
+		let mut duplicate = true;
+
+		if self.valid_votes.insert(valid_vote.1, valid_vote.0).is_none() {
+			self.votes_batched_since_last_tick += 1;
+			duplicate = false;
+		}
+		if self.invalid_votes.insert(invalid_vote.1, invalid_vote.0).is_none() {
+			self.votes_batched_since_last_tick += 1;
+			duplicate = false;
+		}
+
+		if duplicate {
+			Err(pending_response)
+		} else {
+			self.requesters.push((peer, pending_response));
+			Ok(())
+		}
+	}
+
+	/// Check batch for liveness.
+	///
+	/// This function is supposed to be called at instants given at construction and as returned as
+	/// part of `TickResult`.
+	pub(super) fn tick(mut self, now: Instant) -> TickResult {
+		if self.votes_batched_since_last_tick >= MIN_KEEP_BATCH_ALIVE_VOTES &&
+			now < self.best_before
+		{
+			// Still good:
+			let next_tick = self.calculate_next_tick(now);
+			// Reset counter:
+			self.votes_batched_since_last_tick = 0;
+			TickResult::Alive(self, next_tick)
+		} else {
+			TickResult::Done(PreparedImport::from(self))
+		}
+	}
+
+	/// Calculate when the next tick should happen.
+	///
+	/// This will usually return `now + BATCH_COLLECTING_INTERVAL`, except if the lifetime of this
+	/// batch would exceed `MAX_BATCH_LIFETIME`.
+	///
+	/// # Arguments
+	///
+	/// * `now` - The current time.
+	fn calculate_next_tick(&self, now: Instant) -> Instant {
+		let next_tick = now + BATCH_COLLECTING_INTERVAL;
+		if next_tick < self.best_before {
+			next_tick
+		} else {
+			self.best_before
+		}
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/batches/mod.rs

@@ -0,0 +1,170 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{hash_map, HashMap},
+	time::{Duration, Instant},
+};
+
+use futures::future::pending;
+
+use pezkuwi_node_network_protocol::request_response::DISPUTE_REQUEST_TIMEOUT;
+use pezkuwi_primitives::{CandidateHash, CandidateReceiptV2 as CandidateReceipt};
+
+use crate::{
+	receiver::batches::{batch::TickResult, waiting_queue::PendingWake},
+	LOG_TARGET,
+};
+
+pub use self::batch::{Batch, PreparedImport};
+use self::waiting_queue::WaitingQueue;
+
+use super::{
+	error::{JfyiError, JfyiResult},
+	BATCH_COLLECTING_INTERVAL,
+};
+
+/// A single batch (per candidate) as managed by `Batches`.
+mod batch;
+
+/// Queue events in time and wait for them to become ready.
+mod waiting_queue;
+
+/// Safe-guard in case votes trickle in real slow.
+///
+/// If the batch life time exceeded the time the sender is willing to wait for a confirmation, we
+/// would trigger pointless re-sends.
+const MAX_BATCH_LIFETIME: Duration = DISPUTE_REQUEST_TIMEOUT.saturating_sub(Duration::from_secs(2));
+
+/// Limit the number of batches that can be alive at any given time.
+///
+/// Reasoning for this number, see guide.
+pub const MAX_BATCHES: usize = 1000;
+
+/// Manage batches.
+///
+/// - Batches can be found via `find_batch()` in order to add votes to them/check they exist.
+/// - Batches can be checked for being ready for flushing in order to import contained votes.
+pub struct Batches {
+	/// The batches we manage.
+	///
+	/// Kept invariants:
+	/// For each entry in `batches`, there exists an entry in `waiting_queue` as well - we wait on
+	/// all batches!
+	batches: HashMap<CandidateHash, Batch>,
+	/// Waiting queue for waiting for batches to become ready for `tick`.
+	///
+	/// Kept invariants by `Batches`:
+	/// For each entry in the `waiting_queue` there exists a corresponding entry in `batches`.
+	waiting_queue: WaitingQueue<CandidateHash>,
+}
+
+/// A found batch is either really found or got created so it can be found.
+pub enum FoundBatch<'a> {
+	/// Batch just got created.
+	Created(&'a mut Batch),
+	/// Batch already existed.
+	Found(&'a mut Batch),
+}
+
+impl Batches {
+	/// Create new empty `Batches`.
+	pub fn new() -> Self {
+		debug_assert!(
+			MAX_BATCH_LIFETIME > BATCH_COLLECTING_INTERVAL,
+			"Unexpectedly low `MAX_BATCH_LIFETIME`, please check parameters."
+		);
+		Self { batches: HashMap::new(), waiting_queue: WaitingQueue::new() }
+	}
+
+	/// Find a particular batch.
+	///
+	/// That is either find it, or we create it as reflected by the result `FoundBatch`.
+	pub fn find_batch(
+		&mut self,
+		candidate_hash: CandidateHash,
+		candidate_receipt: CandidateReceipt,
+	) -> JfyiResult<FoundBatch<'_>> {
+		if self.batches.len() >= MAX_BATCHES {
+			return Err(JfyiError::MaxBatchLimitReached);
+		}
+		debug_assert!(candidate_hash == candidate_receipt.hash());
+		let result = match self.batches.entry(candidate_hash) {
+			hash_map::Entry::Vacant(vacant) => {
+				let now = Instant::now();
+				let (created, ready_at) = Batch::new(candidate_receipt, now);
+				let pending_wake = PendingWake { payload: candidate_hash, ready_at };
+				self.waiting_queue.push(pending_wake);
+				FoundBatch::Created(vacant.insert(created))
+			},
+			hash_map::Entry::Occupied(occupied) => FoundBatch::Found(occupied.into_mut()),
+		};
+		Ok(result)
+	}
+
+	/// Wait for the next `tick` to check for ready batches.
+	///
+	/// This function blocks (returns `Poll::Pending`) until at least one batch can be
+	/// checked for readiness meaning that `BATCH_COLLECTING_INTERVAL` has passed since the last
+	/// check for that batch or it reached end of life.
+	///
+	/// If this `Batches` instance is empty (does not actually contain any batches), then this
+	/// function will always return `Poll::Pending`.
+	///
+	/// Returns: A `Vec` of all `PreparedImport`s from batches that became ready.
+	pub async fn check_batches(&mut self) -> Vec<PreparedImport> {
+		let now = Instant::now();
+
+		let mut imports = Vec::new();
+
+		// Wait for at least one batch to become ready:
+		self.waiting_queue.wait_ready(now).await;
+
+		// Process all ready entries:
+		while let Some(wake) = self.waiting_queue.pop_ready(now) {
+			let batch = self.batches.remove(&wake.payload);
+			debug_assert!(
+				batch.is_some(),
+				"Entries referenced in `waiting_queue` are supposed to exist!"
+			);
+			let batch = match batch {
+				None => return pending().await,
+				Some(batch) => batch,
+			};
+			match batch.tick(now) {
+				TickResult::Done(import) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?wake.payload,
+						"Batch became ready."
+					);
+					imports.push(import);
+				},
+				TickResult::Alive(old_batch, next_tick) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?wake.payload,
+						"Batch found to be still alive on check."
+					);
+					let pending_wake = PendingWake { payload: wake.payload, ready_at: next_tick };
+					self.waiting_queue.push(pending_wake);
+					self.batches.insert(wake.payload, old_batch);
+				},
+			}
+		}
+		imports
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/batches/waiting_queue.rs

@@ -0,0 +1,204 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{cmp::Ordering, collections::BinaryHeap, time::Instant};
+
+use futures::future::pending;
+use futures_timer::Delay;
+
+/// Wait asynchronously for given `Instant`s one after the other.
+///
+/// `PendingWake`s can be inserted and `WaitingQueue` makes `wait_ready()` to always wait for the
+/// next `Instant` in the queue.
+pub struct WaitingQueue<Payload> {
+	/// All pending wakes we are supposed to wait on in order.
+	pending_wakes: BinaryHeap<PendingWake<Payload>>,
+	/// Wait for next `PendingWake`.
+	timer: Option<Delay>,
+}
+
+/// Represents some event waiting to be processed at `ready_at`.
+///
+/// This is an event in `WaitingQueue`. It provides an `Ord` instance, that sorts descending with
+/// regard to `Instant` (so we get a `min-heap` with the earliest `Instant` at the top).
+#[derive(Eq, PartialEq)]
+pub struct PendingWake<Payload> {
+	pub payload: Payload,
+	pub ready_at: Instant,
+}
+
+impl<Payload: Eq + Ord> WaitingQueue<Payload> {
+	/// Get a new empty `WaitingQueue`.
+	///
+	/// If you call `pop` on this queue immediately, it will always return `Poll::Pending`.
+	pub fn new() -> Self {
+		Self { pending_wakes: BinaryHeap::new(), timer: None }
+	}
+
+	/// Push a `PendingWake`.
+	///
+	/// The next call to `wait_ready` will make sure to wake soon enough to process that new event
+	/// in a timely manner.
+	pub fn push(&mut self, wake: PendingWake<Payload>) {
+		self.pending_wakes.push(wake);
+		// Reset timer as it is potentially obsolete now:
+		self.timer = None;
+	}
+
+	/// Pop the next ready item.
+	///
+	/// This function does not wait, if nothing is ready right now as determined by the passed
+	/// `now` time stamp, this function simply returns `None`.
+	pub fn pop_ready(&mut self, now: Instant) -> Option<PendingWake<Payload>> {
+		let is_ready = self.pending_wakes.peek().map_or(false, |p| p.ready_at <= now);
+		if is_ready {
+			Some(self.pending_wakes.pop().expect("We just peeked. qed."))
+		} else {
+			None
+		}
+	}
+
+	/// Don't pop, just wait until something is ready.
+	///
+	/// Once this function returns `Poll::Ready(())` `pop_ready()` will return `Some`, if passed
+	/// the same `Instant`.
+	///
+	/// Whether ready or not is determined based on the passed time stamp `now` which should be the
+	/// current time as returned by `Instant::now()`
+	///
+	/// This function waits asynchronously for an item to become ready. If there is no more item,
+	/// this call will wait forever (return Poll::Pending without scheduling a wake).
+	pub async fn wait_ready(&mut self, now: Instant) {
+		if let Some(timer) = &mut self.timer {
+			// Previous timer was not done yet.
+			timer.await
+		}
+
+		let next_waiting = self.pending_wakes.peek();
+		let is_ready = next_waiting.map_or(false, |p| p.ready_at <= now);
+		if is_ready {
+			return;
+		}
+
+		self.timer = next_waiting.map(|p| Delay::new(p.ready_at.duration_since(now)));
+		match &mut self.timer {
+			None => return pending().await,
+			Some(timer) => timer.await,
+		}
+	}
+}
+
+impl<Payload: Eq + Ord> PartialOrd<PendingWake<Payload>> for PendingWake<Payload> {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl<Payload: Ord> Ord for PendingWake<Payload> {
+	fn cmp(&self, other: &Self) -> Ordering {
+		// Reverse order for min-heap:
+		match other.ready_at.cmp(&self.ready_at) {
+			Ordering::Equal => other.payload.cmp(&self.payload),
+			o => o,
+		}
+	}
+}
+#[cfg(test)]
+mod tests {
+	use std::{
+		task::Poll,
+		time::{Duration, Instant},
+	};
+
+	use assert_matches::assert_matches;
+	use futures::{future::poll_fn, pin_mut, Future};
+
+	use crate::LOG_TARGET;
+
+	use super::{PendingWake, WaitingQueue};
+
+	#[test]
+	fn wait_ready_waits_for_earliest_event_always() {
+		sp_tracing::try_init_simple();
+		let mut queue = WaitingQueue::new();
+		let now = Instant::now();
+		let start = now;
+		queue.push(PendingWake { payload: 1u32, ready_at: now + Duration::from_millis(3) });
+		// Push another one in order:
+		queue.push(PendingWake { payload: 2u32, ready_at: now + Duration::from_millis(5) });
+		// Push one out of order:
+		queue.push(PendingWake { payload: 0u32, ready_at: now + Duration::from_millis(1) });
+		// Push another one at same timestamp (should become ready at the same time)
+		queue.push(PendingWake { payload: 10u32, ready_at: now + Duration::from_millis(1) });
+
+		futures::executor::block_on(async move {
+			// No time passed yet - nothing should be ready.
+			assert!(queue.pop_ready(now).is_none(), "No time has passed, nothing should be ready");
+
+			// Receive them in order at expected times:
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After first wait.");
+
+			let now = start + Duration::from_millis(1);
+			assert!(Instant::now() - start >= Duration::from_millis(1));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(0u32));
+			// One more should be ready:
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(10u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After second wait.");
+			let now = start + Duration::from_millis(3);
+			assert!(Instant::now() - start >= Duration::from_millis(3));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(1u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			// Push in between wait:
+			poll_fn(|cx| {
+				let fut = queue.wait_ready(now);
+				pin_mut!(fut);
+				assert_matches!(fut.poll(cx), Poll::Pending);
+				Poll::Ready(())
+			})
+			.await;
+			queue.push(PendingWake { payload: 3u32, ready_at: start + Duration::from_millis(4) });
+
+			queue.wait_ready(now).await;
+			// Newly pushed element should have become ready:
+			gum::trace!(target: LOG_TARGET, "After third wait.");
+			let now = start + Duration::from_millis(4);
+			assert!(Instant::now() - start >= Duration::from_millis(4));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(3u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After fourth wait.");
+			let now = start + Duration::from_millis(5);
+			assert!(Instant::now() - start >= Duration::from_millis(5));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(2u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			// queue empty - should wait forever now:
+			poll_fn(|cx| {
+				let fut = queue.wait_ready(now);
+				pin_mut!(fut);
+				assert_matches!(fut.poll(cx), Poll::Pending);
+				Poll::Ready(())
+			})
+			.await;
+		});
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/error.rs

@@ -0,0 +1,97 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Error handling related code and Error/Result definitions.
+
+use fatality::Nested;
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{request_response::incoming, PeerId};
+use pezkuwi_node_subsystem_util::runtime;
+use pezkuwi_primitives::AuthorityDiscoveryId;
+
+use crate::LOG_TARGET;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	#[fatal(forward)]
+	#[error("Retrieving next incoming request failed.")]
+	IncomingRequest(#[from] incoming::Error),
+
+	#[error("Sending back response to peers {0:#?} failed.")]
+	SendResponses(Vec<PeerId>),
+
+	#[error("Changing peer's ({0}) reputation failed.")]
+	SetPeerReputation(PeerId),
+
+	#[error("Dispute request with invalid signatures, from peer {0}.")]
+	InvalidSignature(PeerId),
+
+	#[error("Received votes from peer {0} have been completely redundant.")]
+	RedundantMessage(PeerId),
+
+	#[error("Import of dispute got canceled for candidate {0} - import failed for some reason.")]
+	ImportCanceled(CandidateHash),
+
+	#[error("Peer {0} attempted to participate in dispute and is not a validator.")]
+	NotAValidator(PeerId),
+
+	#[error("Force flush for batch that could not be found attempted, candidate hash: {0}")]
+	ForceFlushBatchDoesNotExist(CandidateHash),
+
+	// Should never happen in practice:
+	#[error("We needed to drop messages, because we reached limit on concurrent batches.")]
+	MaxBatchLimitReached,
+
+	#[error("Authority {0} sent messages at a too high rate.")]
+	AuthorityFlooding(AuthorityDiscoveryId),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub type JfyiResult<T> = std::result::Result<T, JfyiError>;
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them.
+pub fn log_error(result: Result<()>) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Err(error @ JfyiError::ImportCanceled(_)) => {
+			gum::debug!(target: LOG_TARGET, error = ?error);
+			Ok(())
+		},
+		Err(JfyiError::NotAValidator(peer)) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				?peer,
+				"Dropping message from peer (unknown authority id)"
+			);
+			Ok(())
+		},
+		Err(error) => {
+			gum::warn!(target: LOG_TARGET, error = ?error);
+			Ok(())
+		},
+		Ok(()) => Ok(()),
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/mod.rs

@@ -0,0 +1,522 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	pin::Pin,
+	task::{Context, Poll},
+	time::Duration,
+};
+
+use futures::{
+	channel::oneshot,
+	future::poll_fn,
+	pin_mut,
+	stream::{FuturesUnordered, StreamExt},
+	Future,
+};
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{
+	authority_discovery::AuthorityDiscovery,
+	request_response::{
+		incoming::{self, OutgoingResponse, OutgoingResponseSender},
+		v1::{DisputeRequest, DisputeResponse},
+		IncomingRequest, IncomingRequestReceiver,
+	},
+	PeerId, UnifiedReputationChange as Rep,
+};
+use pezkuwi_node_primitives::DISPUTE_WINDOW;
+use pezkuwi_node_subsystem::{
+	messages::{DisputeCoordinatorMessage, ImportStatementsResult},
+	overseer,
+};
+use pezkuwi_node_subsystem_util::{runtime, runtime::RuntimeInfo};
+
+use crate::{
+	metrics::{FAILED, SUCCEEDED},
+	Metrics, LOG_TARGET,
+};
+
+mod error;
+
+/// Rate limiting queues for incoming requests by peers.
+mod peer_queues;
+
+/// Batch imports together.
+mod batches;
+
+use self::{
+	batches::{Batches, FoundBatch, PreparedImport},
+	error::{log_error, JfyiError, JfyiResult, Result},
+	peer_queues::PeerQueues,
+};
+
+const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Received message could not be decoded.");
+const COST_INVALID_SIGNATURE: Rep = Rep::Malicious("Signatures were invalid.");
+const COST_NOT_A_VALIDATOR: Rep = Rep::CostMajor("Reporting peer was not a validator.");
+
+/// Invalid imports can be caused by flooding, e.g. by a disabled validator.
+const COST_INVALID_IMPORT: Rep =
+	Rep::CostMinor("Import was deemed invalid by dispute-coordinator.");
+
+/// How many votes must have arrived in the last `BATCH_COLLECTING_INTERVAL`
+///
+/// in order for a batch to stay alive and not get flushed/imported to the dispute-coordinator.
+///
+/// This ensures a timely import of batches.
+#[cfg(not(test))]
+pub const MIN_KEEP_BATCH_ALIVE_VOTES: u32 = 10;
+#[cfg(test)]
+pub const MIN_KEEP_BATCH_ALIVE_VOTES: u32 = 2;
+
+/// Time we allow to pass for new votes to trickle in.
+///
+/// See `MIN_KEEP_BATCH_ALIVE_VOTES` above.
+/// Should be greater or equal to `RECEIVE_RATE_LIMIT` (there is no point in checking any faster).
+pub const BATCH_COLLECTING_INTERVAL: Duration = Duration::from_millis(500);
+
+/// State for handling incoming `DisputeRequest` messages.
+pub struct DisputesReceiver<Sender, AD> {
+	/// Access to session information.
+	runtime: RuntimeInfo,
+
+	/// Subsystem sender for communication with other subsystems.
+	sender: Sender,
+
+	/// Channel to retrieve incoming requests from.
+	receiver: IncomingRequestReceiver<DisputeRequest>,
+
+	/// Rate limiting queue for each peer (only authorities).
+	peer_queues: PeerQueues,
+
+	/// Currently active batches of imports per candidate.
+	batches: Batches,
+
+	/// Authority discovery service:
+	authority_discovery: AD,
+
+	/// Imports currently being processed by the `dispute-coordinator`.
+	pending_imports: FuturesUnordered<PendingImport>,
+
+	/// Log received requests.
+	metrics: Metrics,
+}
+
+/// Messages as handled by this receiver internally.
+enum MuxedMessage {
+	/// An import got confirmed by the coordinator.
+	///
+	/// We need to handle those for two reasons:
+	///
+	/// - We need to make sure responses are actually sent (therefore we need to await futures
+	/// promptly).
+	/// - We need to punish peers whose import got rejected.
+	ConfirmedImport(ImportResult),
+
+	/// A new request has arrived and should be handled.
+	NewRequest(IncomingRequest<DisputeRequest>),
+
+	/// Rate limit timer hit - is time to process one row of messages.
+	///
+	/// This is the result of calling `self.peer_queues.pop_reqs()`.
+	WakePeerQueuesPopReqs(Vec<IncomingRequest<DisputeRequest>>),
+
+	/// It is time to check batches.
+	///
+	/// Every `BATCH_COLLECTING_INTERVAL` we check whether less than `MIN_KEEP_BATCH_ALIVE_VOTES`
+	/// new votes arrived, if so the batch is ready for import.
+	///
+	/// This is the result of calling `self.batches.check_batches()`.
+	WakeCheckBatches(Vec<PreparedImport>),
+}
+
+impl<Sender, AD> DisputesReceiver<Sender, AD>
+where
+	AD: AuthorityDiscovery,
+	Sender: overseer::DisputeDistributionSenderTrait,
+{
+	/// Create a new receiver which can be `run`.
+	pub fn new(
+		sender: Sender,
+		receiver: IncomingRequestReceiver<DisputeRequest>,
+		authority_discovery: AD,
+		metrics: Metrics,
+	) -> Self {
+		let runtime = RuntimeInfo::new_with_config(runtime::Config {
+			keystore: None,
+			session_cache_lru_size: DISPUTE_WINDOW.get(),
+		});
+		Self {
+			runtime,
+			sender,
+			receiver,
+			peer_queues: PeerQueues::new(),
+			batches: Batches::new(),
+			authority_discovery,
+			pending_imports: FuturesUnordered::new(),
+			metrics,
+		}
+	}
+
+	/// Get that receiver started.
+	///
+	/// This is an endless loop and should be spawned into its own task.
+	pub async fn run(mut self) {
+		loop {
+			match log_error(self.run_inner().await) {
+				Ok(()) => {},
+				Err(fatal) => {
+					gum::debug!(
+						target: LOG_TARGET,
+						error = ?fatal,
+						"Shutting down"
+					);
+					return;
+				},
+			}
+		}
+	}
+
+	/// Actual work happening here in three phases:
+	///
+	/// 1. Receive and queue incoming messages until the rate limit timer hits.
+	/// 2. Do import/batching for the head of all queues.
+	/// 3. Check and flush any ready batches.
+	async fn run_inner(&mut self) -> Result<()> {
+		let msg = self.receive_message().await?;
+
+		match msg {
+			MuxedMessage::NewRequest(req) => {
+				// Phase 1:
+				self.metrics.on_received_request();
+				self.dispatch_to_queues(req).await?;
+			},
+			MuxedMessage::WakePeerQueuesPopReqs(reqs) => {
+				// Phase 2:
+				for req in reqs {
+					// No early return - we cannot cancel imports of one peer, because the import of
+					// another failed:
+					match log_error(self.start_import_or_batch(req).await) {
+						Ok(()) => {},
+						Err(fatal) => return Err(fatal.into()),
+					}
+				}
+			},
+			MuxedMessage::WakeCheckBatches(ready_imports) => {
+				// Phase 3:
+				self.import_ready_batches(ready_imports).await;
+			},
+			MuxedMessage::ConfirmedImport(import_result) => {
+				self.update_imported_requests_metrics(&import_result);
+				// Confirm imports to requesters/punish them on invalid imports:
+				send_responses_to_requesters(import_result).await?;
+			},
+		}
+
+		Ok(())
+	}
+
+	/// Receive one `MuxedMessage`.
+	///
+	///
+	/// Dispatching events to messages as they happen.
+	async fn receive_message(&mut self) -> Result<MuxedMessage> {
+		poll_fn(|ctx| {
+			// In case of Ready(None), we want to wait for pending requests:
+			if let Poll::Ready(Some(v)) = self.pending_imports.poll_next_unpin(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::ConfirmedImport(v?)));
+			}
+
+			let rate_limited = self.peer_queues.pop_reqs();
+			pin_mut!(rate_limited);
+			// We poll rate_limit before batches, so we don't unnecessarily delay importing to
+			// batches.
+			if let Poll::Ready(reqs) = rate_limited.poll(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::WakePeerQueuesPopReqs(reqs)));
+			}
+
+			let ready_batches = self.batches.check_batches();
+			pin_mut!(ready_batches);
+			if let Poll::Ready(ready_batches) = ready_batches.poll(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::WakeCheckBatches(ready_batches)));
+			}
+
+			let next_req = self.receiver.recv(|| vec![COST_INVALID_REQUEST]);
+			pin_mut!(next_req);
+			if let Poll::Ready(r) = next_req.poll(ctx) {
+				return match r {
+					Err(e) => Poll::Ready(Err(incoming::Error::from(e).into())),
+					Ok(v) => Poll::Ready(Ok(MuxedMessage::NewRequest(v))),
+				};
+			}
+			Poll::Pending
+		})
+		.await
+	}
+
+	/// Process incoming requests.
+	///
+	/// - Check sender is authority
+	/// - Dispatch message to corresponding queue in `peer_queues`.
+	/// - If queue is full, drop message and change reputation of sender.
+	async fn dispatch_to_queues(&mut self, req: IncomingRequest<DisputeRequest>) -> JfyiResult<()> {
+		let peer = req.peer;
+		// Only accept messages from validators, in case there are multiple `AuthorityId`s, we
+		// just take the first one. On session boundaries this might allow validators to double
+		// their rate limit for a short period of time, which seems acceptable.
+		let authority_id = match self
+			.authority_discovery
+			.get_authority_ids_by_peer_id(peer)
+			.await
+			.and_then(|s| s.into_iter().next())
+		{
+			None => {
+				req.send_outgoing_response(OutgoingResponse {
+					result: Err(()),
+					reputation_changes: vec![COST_NOT_A_VALIDATOR],
+					sent_feedback: None,
+				})
+				.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+				return Err(JfyiError::NotAValidator(peer).into());
+			},
+			Some(auth_id) => auth_id,
+		};
+
+		// Queue request:
+		if let Err((authority_id, req)) = self.peer_queues.push_req(authority_id, req) {
+			gum::debug!(
+				target: LOG_TARGET,
+				?authority_id,
+				?peer,
+				"Peer hit the rate limit - dropping message."
+			);
+			req.send_outgoing_response(OutgoingResponse {
+				result: Err(()),
+				reputation_changes: vec![],
+				sent_feedback: None,
+			})
+			.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+			return Err(JfyiError::AuthorityFlooding(authority_id));
+		}
+		Ok(())
+	}
+
+	/// Start importing votes for the given request or batch.
+	///
+	/// Signature check and in case we already have an existing batch we import to that batch,
+	/// otherwise import to `dispute-coordinator` directly and open a batch.
+	async fn start_import_or_batch(
+		&mut self,
+		incoming: IncomingRequest<DisputeRequest>,
+	) -> Result<()> {
+		let IncomingRequest { peer, payload, pending_response } = incoming;
+
+		let info = self
+			.runtime
+			.get_session_info_by_index(
+				&mut self.sender,
+				payload.0.candidate_receipt.descriptor.relay_parent(),
+				payload.0.session_index,
+			)
+			.await?;
+
+		let votes_result = payload.0.try_into_signed_votes(&info.session_info);
+
+		let (candidate_receipt, valid_vote, invalid_vote) = match votes_result {
+			Err(()) => {
+				// Signature invalid:
+				pending_response
+					.send_outgoing_response(OutgoingResponse {
+						result: Err(()),
+						reputation_changes: vec![COST_INVALID_SIGNATURE],
+						sent_feedback: None,
+					})
+					.map_err(|_| JfyiError::SetPeerReputation(peer))?;
+
+				return Err(From::from(JfyiError::InvalidSignature(peer)));
+			},
+			Ok(votes) => votes,
+		};
+
+		let candidate_hash = *valid_vote.0.candidate_hash();
+
+		match self.batches.find_batch(candidate_hash, candidate_receipt)? {
+			FoundBatch::Created(batch) => {
+				// There was no entry yet - start import immediately:
+				gum::trace!(
+					target: LOG_TARGET,
+					?candidate_hash,
+					?peer,
+					"No batch yet - triggering immediate import"
+				);
+				let import = PreparedImport {
+					candidate_receipt: batch.candidate_receipt().clone(),
+					statements: vec![valid_vote, invalid_vote],
+					requesters: vec![(peer, pending_response)],
+				};
+				self.start_import(import).await;
+			},
+			FoundBatch::Found(batch) => {
+				gum::trace!(target: LOG_TARGET, ?candidate_hash, "Batch exists - batching request");
+				let batch_result =
+					batch.add_votes(valid_vote, invalid_vote, peer, pending_response);
+
+				if let Err(pending_response) = batch_result {
+					// We don't expect honest peers to send redundant votes within a single batch,
+					// as the timeout for retry is much higher. Still we don't want to punish the
+					// node as it might not be the node's fault. Some other (malicious) node could
+					// have been faster sending the same votes in order to harm the reputation of
+					// that honest node. Given that we already have a rate limit, if a validator
+					// chooses to waste available rate with redundant votes - so be it. The actual
+					// dispute resolution is unaffected.
+					gum::debug!(
+						target: LOG_TARGET,
+						?peer,
+						"Peer sent completely redundant votes within a single batch - that looks fishy!",
+					);
+					pending_response
+						.send_outgoing_response(OutgoingResponse {
+							// While we have seen duplicate votes, we cannot confirm as we don't
+							// know yet whether the batch is going to be confirmed, so we assume
+							// the worst. We don't want to push the pending response to the batch
+							// either as that would be unbounded, only limited by the rate limit.
+							result: Err(()),
+							reputation_changes: Vec::new(),
+							sent_feedback: None,
+						})
+						.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+					return Err(From::from(JfyiError::RedundantMessage(peer)));
+				}
+			},
+		}
+
+		Ok(())
+	}
+
+	/// Trigger import into the dispute-coordinator of ready batches (`PreparedImport`s).
+	async fn import_ready_batches(&mut self, ready_imports: Vec<PreparedImport>) {
+		for import in ready_imports {
+			self.start_import(import).await;
+		}
+	}
+
+	/// Start import and add response receiver to `pending_imports`.
+	async fn start_import(&mut self, import: PreparedImport) {
+		let PreparedImport { candidate_receipt, statements, requesters } = import;
+		let (session_index, candidate_hash) = match statements.iter().next() {
+			None => {
+				gum::debug!(
+					target: LOG_TARGET,
+					candidate_hash = ?candidate_receipt.hash(),
+					"Not importing empty batch"
+				);
+				return;
+			},
+			Some(vote) => (vote.0.session_index(), *vote.0.candidate_hash()),
+		};
+
+		let (pending_confirmation, confirmation_rx) = oneshot::channel();
+		self.sender
+			.send_message(DisputeCoordinatorMessage::ImportStatements {
+				candidate_receipt,
+				session: session_index,
+				statements,
+				pending_confirmation: Some(pending_confirmation),
+			})
+			.await;
+
+		let pending =
+			PendingImport { candidate_hash, requesters, pending_response: confirmation_rx };
+
+		self.pending_imports.push(pending);
+	}
+
+	fn update_imported_requests_metrics(&self, result: &ImportResult) {
+		let label = match result.result {
+			ImportStatementsResult::ValidImport => SUCCEEDED,
+			ImportStatementsResult::InvalidImport => FAILED,
+		};
+		self.metrics.on_imported(label, result.requesters.len());
+	}
+}
+
+async fn send_responses_to_requesters(import_result: ImportResult) -> JfyiResult<()> {
+	let ImportResult { requesters, result } = import_result;
+
+	let mk_response = match result {
+		ImportStatementsResult::ValidImport => || OutgoingResponse {
+			result: Ok(DisputeResponse::Confirmed),
+			reputation_changes: Vec::new(),
+			sent_feedback: None,
+		},
+		ImportStatementsResult::InvalidImport => || OutgoingResponse {
+			result: Err(()),
+			reputation_changes: vec![COST_INVALID_IMPORT],
+			sent_feedback: None,
+		},
+	};
+
+	let mut sending_failed_for = Vec::new();
+	for (peer, pending_response) in requesters {
+		if let Err(()) = pending_response.send_outgoing_response(mk_response()) {
+			sending_failed_for.push(peer);
+		}
+	}
+
+	if !sending_failed_for.is_empty() {
+		Err(JfyiError::SendResponses(sending_failed_for))
+	} else {
+		Ok(())
+	}
+}
+
+/// A future that resolves into an `ImportResult` when ready.
+///
+/// This future is used on `dispute-coordinator` import messages for the oneshot response receiver
+/// to:
+/// - Keep track of concerned `CandidateHash` for reporting errors.
+/// - Keep track of requesting peers so we can confirm the import/punish them on invalid imports.
+struct PendingImport {
+	candidate_hash: CandidateHash,
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+	pending_response: oneshot::Receiver<ImportStatementsResult>,
+}
+
+/// A `PendingImport` becomes an `ImportResult` once done.
+struct ImportResult {
+	/// Requesters of that import.
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+	/// Actual result of the import.
+	result: ImportStatementsResult,
+}
+
+impl PendingImport {
+	async fn wait_for_result(&mut self) -> JfyiResult<ImportResult> {
+		let result = (&mut self.pending_response)
+			.await
+			.map_err(|_| JfyiError::ImportCanceled(self.candidate_hash))?;
+		Ok(ImportResult { requesters: std::mem::take(&mut self.requesters), result })
+	}
+}
+
+impl Future for PendingImport {
+	type Output = JfyiResult<ImportResult>;
+	fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+		let fut = self.wait_for_result();
+		pin_mut!(fut);
+		fut.poll(cx)
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/peer_queues.rs

@@ -0,0 +1,141 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{hash_map::Entry, HashMap, VecDeque};
+
+use futures::future::pending;
+use futures_timer::Delay;
+use pezkuwi_node_network_protocol::request_response::{v1::DisputeRequest, IncomingRequest};
+use pezkuwi_primitives::AuthorityDiscoveryId;
+
+use crate::RECEIVE_RATE_LIMIT;
+
+/// How many messages we are willing to queue per peer (validator).
+///
+/// The larger this value is, the larger bursts are allowed to be without us dropping messages. On
+/// the flip side this gets allocated per validator, so for a size of 10 this will result
+/// in `10_000 * size_of(IncomingRequest)` in the worst case.
+///
+/// `PEER_QUEUE_CAPACITY` must not be 0 for obvious reasons.
+#[cfg(not(test))]
+pub const PEER_QUEUE_CAPACITY: usize = 10;
+#[cfg(test)]
+pub const PEER_QUEUE_CAPACITY: usize = 2;
+
+/// Queues for messages from authority peers for rate limiting.
+///
+/// Invariants ensured:
+///
+/// 1. No queue will ever have more than `PEER_QUEUE_CAPACITY` elements.
+/// 2. There are no empty queues. Whenever a queue gets empty, it is removed. This way checking
+///    whether there are any messages queued is cheap.
+/// 3. As long as not empty, `pop_reqs` will, if called in sequence, not return `Ready` more often
+///    than once for every `RECEIVE_RATE_LIMIT`, but it will always return Ready eventually.
+/// 4. If empty `pop_reqs` will never return `Ready`, but will always be `Pending`.
+pub struct PeerQueues {
+	/// Actual queues.
+	queues: HashMap<AuthorityDiscoveryId, VecDeque<IncomingRequest<DisputeRequest>>>,
+
+	/// Delay timer for establishing the rate limit.
+	rate_limit_timer: Option<Delay>,
+}
+
+impl PeerQueues {
+	/// New empty `PeerQueues`.
+	pub fn new() -> Self {
+		Self { queues: HashMap::new(), rate_limit_timer: None }
+	}
+
+	/// Push an incoming request for a given authority.
+	///
+	/// Returns: `Ok(())` if succeeded, `Err((args))` if capacity is reached.
+	pub fn push_req(
+		&mut self,
+		peer: AuthorityDiscoveryId,
+		req: IncomingRequest<DisputeRequest>,
+	) -> Result<(), (AuthorityDiscoveryId, IncomingRequest<DisputeRequest>)> {
+		let queue = match self.queues.entry(peer) {
+			Entry::Vacant(vacant) => vacant.insert(VecDeque::new()),
+			Entry::Occupied(occupied) => {
+				if occupied.get().len() >= PEER_QUEUE_CAPACITY {
+					return Err((occupied.key().clone(), req));
+				}
+				occupied.into_mut()
+			},
+		};
+		queue.push_back(req);
+
+		// We have at least one element to process - rate limit `timer` needs to exist now:
+		self.ensure_timer();
+		Ok(())
+	}
+
+	/// Pop all heads and return them for processing.
+	///
+	/// This gets one message from each peer that has sent at least one.
+	///
+	/// This function is rate limited, if called in sequence it will not return more often than
+	/// every `RECEIVE_RATE_LIMIT`.
+	///
+	/// NOTE: If empty this function will not return `Ready` at all, but will always be `Pending`.
+	pub async fn pop_reqs(&mut self) -> Vec<IncomingRequest<DisputeRequest>> {
+		self.wait_for_timer().await;
+
+		let mut heads = Vec::with_capacity(self.queues.len());
+		let old_queues = std::mem::replace(&mut self.queues, HashMap::new());
+		for (k, mut queue) in old_queues.into_iter() {
+			let front = queue.pop_front();
+			debug_assert!(front.is_some(), "Invariant that queues are never empty is broken.");
+
+			if let Some(front) = front {
+				heads.push(front);
+			}
+			if !queue.is_empty() {
+				self.queues.insert(k, queue);
+			}
+		}
+
+		if !self.is_empty() {
+			// Still not empty - we should get woken at some point.
+			self.ensure_timer();
+		}
+
+		heads
+	}
+
+	/// Whether or not all queues are empty.
+	pub fn is_empty(&self) -> bool {
+		self.queues.is_empty()
+	}
+
+	/// Ensure there is an active `timer`.
+	///
+	/// Checks whether one exists and if not creates one.
+	fn ensure_timer(&mut self) -> &mut Delay {
+		self.rate_limit_timer.get_or_insert(Delay::new(RECEIVE_RATE_LIMIT))
+	}
+
+	/// Wait for `timer` if it exists, or be `Pending` forever.
+	///
+	/// Afterwards it gets set back to `None`.
+	async fn wait_for_timer(&mut self) {
+		match self.rate_limit_timer.as_mut() {
+			None => pending().await,
+			Some(timer) => timer.await,
+		}
+		self.rate_limit_timer = None;
+	}
+}