feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/node/network/dispute-distribution/Cargo.toml

@@ -0,0 +1,53 @@
+[package]
+name = "pezkuwi-dispute-distribution"
+version = "7.0.0"
+description = "Pezkuwi Dispute Distribution subsystem, which ensures all concerned validators are aware of a dispute and have the relevant votes."
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+codec = { features = ["std"], workspace = true, default-features = true }
+fatality = { workspace = true }
+futures = { workspace = true }
+futures-timer = { workspace = true }
+gum = { workspace = true, default-features = true }
+indexmap = { workspace = true }
+pezkuwi-node-network-protocol = { workspace = true, default-features = true }
+pezkuwi-node-primitives = { workspace = true, default-features = true }
+pezkuwi-node-subsystem = { workspace = true, default-features = true }
+pezkuwi-node-subsystem-util = { workspace = true, default-features = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+sc-network = { workspace = true, default-features = true }
+sp-application-crypto = { workspace = true, default-features = true }
+sp-keystore = { workspace = true, default-features = true }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+assert_matches = { workspace = true }
+async-channel = { workspace = true }
+async-trait = { workspace = true }
+pezkuwi-node-subsystem-test-helpers = { workspace = true }
+pezkuwi-primitives-test-helpers = { workspace = true }
+sc-keystore = { workspace = true, default-features = true }
+sp-keyring = { workspace = true, default-features = true }
+sp-tracing = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"gum/runtime-benchmarks",
+	"pezkuwi-node-network-protocol/runtime-benchmarks",
+	"pezkuwi-node-primitives/runtime-benchmarks",
+	"pezkuwi-node-subsystem-test-helpers/runtime-benchmarks",
+	"pezkuwi-node-subsystem-util/runtime-benchmarks",
+	"pezkuwi-node-subsystem/runtime-benchmarks",
+	"pezkuwi-primitives-test-helpers/runtime-benchmarks",
+	"pezkuwi-primitives/runtime-benchmarks",
+	"sc-network/runtime-benchmarks",
+	"sp-keyring/runtime-benchmarks",
+]

pezkuwi/node/network/dispute-distribution/src/error.rs

@@ -0,0 +1,72 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Error handling related code and Error/Result definitions.
+
+use pezkuwi_node_subsystem::SubsystemError;
+use pezkuwi_node_subsystem_util::runtime;
+
+use crate::{sender, LOG_TARGET};
+
+use fatality::Nested;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	/// Receiving subsystem message from overseer failed.
+	#[fatal]
+	#[error("Receiving message from overseer failed")]
+	SubsystemReceive(#[source] SubsystemError),
+
+	/// Spawning a running task failed.
+	#[fatal]
+	#[error("Spawning subsystem task failed")]
+	SpawnTask(#[source] SubsystemError),
+
+	/// `DisputeSender` mpsc receiver exhausted.
+	#[fatal]
+	#[error("Erasure chunk requester stream exhausted")]
+	SenderExhausted,
+
+	/// Errors coming from `runtime::Runtime`.
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	/// Errors coming from `DisputeSender`
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Sender(#[from] sender::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub type FatalResult<T> = std::result::Result<T, FatalError>;
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them
+pub fn log_error(result: Result<()>, ctx: &'static str) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Err(jfyi) => {
+			gum::warn!(target: LOG_TARGET, error = ?jfyi, ctx);
+			Ok(())
+		},
+		Ok(()) => Ok(()),
+	}
+}

pezkuwi/node/network/dispute-distribution/src/lib.rs

@@ -0,0 +1,297 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+//! # Sending and receiving of `DisputeRequest`s.
+//!
+//! This subsystem essentially consists of two parts:
+//!
+//! - a sender
+//! - and a receiver
+//!
+//! The sender is responsible for getting our vote out, see `sender`. The receiver handles
+//! incoming [`DisputeRequest`](v1::DisputeRequest)s and offers spam protection, see `receiver`.
+
+use std::time::Duration;
+
+use futures::{channel::mpsc, FutureExt, StreamExt, TryFutureExt};
+
+use pezkuwi_node_network_protocol::authority_discovery::AuthorityDiscovery;
+use pezkuwi_node_subsystem_util::nesting_sender::NestingSender;
+use sp_keystore::KeystorePtr;
+
+use pezkuwi_node_network_protocol::request_response::{incoming::IncomingRequestReceiver, v1};
+use pezkuwi_node_primitives::DISPUTE_WINDOW;
+use pezkuwi_node_subsystem::{
+	messages::DisputeDistributionMessage, overseer, FromOrchestra, OverseerSignal,
+	SpawnedSubsystem, SubsystemError,
+};
+use pezkuwi_node_subsystem_util::{runtime, runtime::RuntimeInfo};
+
+/// ## The sender [`DisputeSender`]
+///
+/// The sender (`DisputeSender`) keeps track of live disputes and makes sure our vote gets out for
+/// each one of those. The sender is responsible for sending our vote to each validator
+/// participating in the dispute and to each authority currently authoring blocks. The sending can
+/// be initiated by sending `DisputeDistributionMessage::SendDispute` message to this subsystem.
+///
+/// In addition the `DisputeSender` will query the coordinator for active disputes on each
+/// [`DisputeSender::update_leaves`] call and will initiate sending (start a `SendTask`) for every,
+/// to this subsystem, unknown dispute. This is to make sure, we get our vote out, even on
+/// restarts.
+///
+/// The actual work of sending and keeping track of transmission attempts to each validator for a
+/// particular dispute are done by [`SendTask`].  The purpose of the `DisputeSender` is to keep
+/// track of all ongoing disputes and start and clean up `SendTask`s accordingly.
+mod sender;
+use self::sender::{DisputeSender, DisputeSenderMessage};
+
+/// ## The receiver [`DisputesReceiver`]
+///
+/// The receiving side is implemented as `DisputesReceiver` and is run as a separate long running
+/// task within this subsystem ([`DisputesReceiver::run`]).
+///
+/// Conceptually all the receiver has to do, is waiting for incoming requests which are passed in
+/// via a dedicated channel and forwarding them to the dispute coordinator via
+/// `DisputeCoordinatorMessage::ImportStatements`. Being the interface to the network and untrusted
+/// nodes, the reality is not that simple of course. Before importing statements the receiver will
+/// batch up imports as well as possible for efficient imports while maintaining timely dispute
+/// resolution and handling of spamming validators:
+///
+/// - Drop all messages from non validator nodes, for this it requires the [`AuthorityDiscovery`]
+/// service.
+/// - Drop messages from a node, if it sends at a too high rate.
+/// - Filter out duplicate messages (over some period of time).
+/// - Drop any obviously invalid votes (invalid signatures for example).
+/// - Ban peers whose votes were deemed invalid.
+///
+/// In general dispute-distribution works on limiting the work the dispute-coordinator will have to
+/// do, while at the same time making it aware of new disputes as fast as possible.
+///
+/// For successfully imported votes, we will confirm the receipt of the message back to the sender.
+/// This way a received confirmation guarantees, that the vote has been stored to disk by the
+/// receiver.
+mod receiver;
+use self::receiver::DisputesReceiver;
+
+/// Error and [`Result`] type for this subsystem.
+mod error;
+use error::{log_error, Error, FatalError, FatalResult, Result};
+
+#[cfg(test)]
+mod tests;
+
+mod metrics;
+//// Prometheus `Metrics` for dispute distribution.
+pub use metrics::Metrics;
+
+const LOG_TARGET: &'static str = "teyrchain::dispute-distribution";
+
+/// Rate limit on the `receiver` side.
+///
+/// If messages from one peer come in at a higher rate than every `RECEIVE_RATE_LIMIT` on average,
+/// we start dropping messages from that peer to enforce that limit.
+pub const RECEIVE_RATE_LIMIT: Duration = Duration::from_millis(100);
+
+/// Rate limit on the `sender` side.
+///
+/// In order to not hit the `RECEIVE_RATE_LIMIT` on the receiving side, we limit out sending rate as
+/// well.
+///
+/// We add 50ms extra, just to have some save margin to the `RECEIVE_RATE_LIMIT`.
+pub const SEND_RATE_LIMIT: Duration = RECEIVE_RATE_LIMIT.saturating_add(Duration::from_millis(50));
+
+/// The dispute distribution subsystem.
+pub struct DisputeDistributionSubsystem<AD> {
+	/// Easy and efficient runtime access for this subsystem.
+	runtime: RuntimeInfo,
+
+	/// Sender for our dispute requests.
+	disputes_sender: DisputeSender<DisputeSenderMessage>,
+
+	/// Receive messages from `DisputeSender` background tasks.
+	sender_rx: mpsc::Receiver<DisputeSenderMessage>,
+
+	/// Receiver for incoming requests.
+	req_receiver: Option<IncomingRequestReceiver<v1::DisputeRequest>>,
+
+	/// Authority discovery service.
+	authority_discovery: AD,
+
+	/// Metrics for this subsystem.
+	metrics: Metrics,
+}
+
+#[overseer::subsystem(DisputeDistribution, error = SubsystemError, prefix = self::overseer)]
+impl<Context, AD> DisputeDistributionSubsystem<AD>
+where
+	<Context as overseer::DisputeDistributionContextTrait>::Sender:
+		overseer::DisputeDistributionSenderTrait + Sync + Send,
+	AD: AuthorityDiscovery + Clone,
+{
+	fn start(self, ctx: Context) -> SpawnedSubsystem {
+		let future = self
+			.run(ctx)
+			.map_err(|e| SubsystemError::with_origin("dispute-distribution", e))
+			.boxed();
+
+		SpawnedSubsystem { name: "dispute-distribution-subsystem", future }
+	}
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl<AD> DisputeDistributionSubsystem<AD>
+where
+	AD: AuthorityDiscovery + Clone,
+{
+	/// Create a new instance of the dispute distribution.
+	pub fn new(
+		keystore: KeystorePtr,
+		req_receiver: IncomingRequestReceiver<v1::DisputeRequest>,
+		authority_discovery: AD,
+		metrics: Metrics,
+	) -> Self {
+		let runtime = RuntimeInfo::new_with_config(runtime::Config {
+			keystore: Some(keystore),
+			session_cache_lru_size: DISPUTE_WINDOW.get(),
+		});
+		let (tx, sender_rx) = NestingSender::new_root(1);
+		let disputes_sender = DisputeSender::new(tx, metrics.clone());
+		Self {
+			runtime,
+			disputes_sender,
+			sender_rx,
+			req_receiver: Some(req_receiver),
+			authority_discovery,
+			metrics,
+		}
+	}
+
+	/// Start processing work as passed on from the Overseer.
+	async fn run<Context>(mut self, mut ctx: Context) -> std::result::Result<(), FatalError> {
+		let receiver = DisputesReceiver::new(
+			ctx.sender().clone(),
+			self.req_receiver
+				.take()
+				.expect("Must be provided on `new` and we take ownership here. qed."),
+			self.authority_discovery.clone(),
+			self.metrics.clone(),
+		);
+		ctx.spawn("disputes-receiver", receiver.run().boxed())
+			.map_err(FatalError::SpawnTask)?;
+
+		// Process messages for sending side.
+		//
+		// Note: We want the sender to be rate limited and we are currently taking advantage of the
+		// fact that the root task of this subsystem is only concerned with sending: Functions of
+		// `DisputeSender` might back pressure if the rate limit is hit, which will slow down this
+		// loop. If this fact ever changes, we will likely need another task.
+		loop {
+			let message = MuxedMessage::receive(&mut ctx, &mut self.sender_rx).await;
+			match message {
+				MuxedMessage::Subsystem(result) => {
+					let result = match result? {
+						FromOrchestra::Signal(signal) => {
+							match self.handle_signals(&mut ctx, signal).await {
+								Ok(SignalResult::Conclude) => return Ok(()),
+								Ok(SignalResult::Continue) => Ok(()),
+								Err(f) => Err(f),
+							}
+						},
+						FromOrchestra::Communication { msg } =>
+							self.handle_subsystem_message(&mut ctx, msg).await,
+					};
+					log_error(result, "on FromOrchestra")?;
+				},
+				MuxedMessage::Sender(result) => {
+					let result = self
+						.disputes_sender
+						.on_message(
+							&mut ctx,
+							&mut self.runtime,
+							result.ok_or(FatalError::SenderExhausted)?,
+						)
+						.await
+						.map_err(Error::Sender);
+					log_error(result, "on_message")?;
+				},
+			}
+		}
+	}
+
+	/// Handle overseer signals.
+	async fn handle_signals<Context>(
+		&mut self,
+		ctx: &mut Context,
+		signal: OverseerSignal,
+	) -> Result<SignalResult> {
+		match signal {
+			OverseerSignal::Conclude => return Ok(SignalResult::Conclude),
+			OverseerSignal::ActiveLeaves(update) => {
+				self.disputes_sender.update_leaves(ctx, &mut self.runtime, update).await?;
+			},
+			OverseerSignal::BlockFinalized(_, _) => {},
+		};
+		Ok(SignalResult::Continue)
+	}
+
+	/// Handle `DisputeDistributionMessage`s.
+	async fn handle_subsystem_message<Context>(
+		&mut self,
+		ctx: &mut Context,
+		msg: DisputeDistributionMessage,
+	) -> Result<()> {
+		match msg {
+			DisputeDistributionMessage::SendDispute(dispute_msg) =>
+				self.disputes_sender.start_sender(ctx, &mut self.runtime, dispute_msg).await?,
+		}
+		Ok(())
+	}
+}
+
+/// Messages to be handled in this subsystem.
+#[derive(Debug)]
+enum MuxedMessage {
+	/// Messages from other subsystems.
+	Subsystem(FatalResult<FromOrchestra<DisputeDistributionMessage>>),
+	/// Messages from spawned sender background tasks.
+	Sender(Option<DisputeSenderMessage>),
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl MuxedMessage {
+	async fn receive<Context>(
+		ctx: &mut Context,
+		from_sender: &mut mpsc::Receiver<DisputeSenderMessage>,
+	) -> Self {
+		// We are only fusing here to make `select` happy, in reality we will quit if the stream
+		// ends.
+		let from_overseer = ctx.recv().fuse();
+		futures::pin_mut!(from_overseer, from_sender);
+		// We select biased to make sure we finish up loose ends, before starting new work.
+		futures::select_biased!(
+			msg = from_sender.next() => MuxedMessage::Sender(msg),
+			msg = from_overseer => MuxedMessage::Subsystem(msg.map_err(FatalError::SubsystemReceive)),
+		)
+	}
+}
+
+/// Result of handling signal from overseer.
+enum SignalResult {
+	/// Overseer asked us to conclude.
+	Conclude,
+	/// We can continue processing events.
+	Continue,
+}

pezkuwi/node/network/dispute-distribution/src/metrics.rs

@@ -0,0 +1,130 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use pezkuwi_node_subsystem_util::{
+	metrics,
+	metrics::{
+		prometheus,
+		prometheus::{Counter, CounterVec, Opts, PrometheusError, Registry, U64},
+	},
+};
+
+/// Label for success counters.
+pub const SUCCEEDED: &'static str = "succeeded";
+
+/// Label for fail counters.
+pub const FAILED: &'static str = "failed";
+
+/// Dispute Distribution metrics.
+#[derive(Clone, Default)]
+pub struct Metrics(Option<MetricsInner>);
+
+#[derive(Clone)]
+struct MetricsInner {
+	/// Number of sent dispute requests (succeeded and failed).
+	sent_requests: CounterVec<U64>,
+
+	/// Number of requests received.
+	///
+	/// This is all requests coming in, regardless of whether they are processed or dropped.
+	received_requests: Counter<U64>,
+
+	/// Number of requests for which `ImportStatements` returned.
+	///
+	/// We both have successful imports and failed imports here.
+	imported_requests: CounterVec<U64>,
+
+	/// The duration of issued dispute request to response.
+	time_dispute_request: prometheus::Histogram,
+}
+
+impl Metrics {
+	/// Create new dummy metrics, not reporting anything.
+	pub fn new_dummy() -> Self {
+		Metrics(None)
+	}
+
+	/// Increment counter on finished request sending.
+	pub fn on_sent_request(&self, label: &'static str) {
+		if let Some(metrics) = &self.0 {
+			metrics.sent_requests.with_label_values(&[label]).inc()
+		}
+	}
+
+	/// Increment counter on served disputes.
+	pub fn on_received_request(&self) {
+		if let Some(metrics) = &self.0 {
+			metrics.received_requests.inc()
+		}
+	}
+
+	/// Statements have been imported.
+	pub fn on_imported(&self, label: &'static str, num_requests: usize) {
+		if let Some(metrics) = &self.0 {
+			metrics
+				.imported_requests
+				.with_label_values(&[label])
+				.inc_by(num_requests as u64)
+		}
+	}
+
+	/// Get a timer to time request/response duration.
+	pub fn time_dispute_request(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.time_dispute_request.start_timer())
+	}
+}
+
+impl metrics::Metrics for Metrics {
+	fn try_register(registry: &Registry) -> Result<Self, PrometheusError> {
+		let metrics = MetricsInner {
+			sent_requests: prometheus::register(
+				CounterVec::new(
+					Opts::new(
+						"pezkuwi_teyrchain_dispute_distribution_sent_requests",
+						"Total number of sent requests.",
+					),
+					&["success"],
+				)?,
+				registry,
+			)?,
+			received_requests: prometheus::register(
+				Counter::new(
+					"pezkuwi_teyrchain_dispute_distribution_received_requests",
+					"Total number of received dispute requests.",
+				)?,
+				registry,
+			)?,
+			imported_requests: prometheus::register(
+				CounterVec::new(
+					Opts::new(
+						"pezkuwi_teyrchain_dispute_distribution_imported_requests",
+						"Total number of imported requests.",
+					),
+					&["success"],
+				)?,
+				registry,
+			)?,
+			time_dispute_request: prometheus::register(
+				prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
+					"pezkuwi_teyrchain_dispute_distribution_time_dispute_request",
+					"Time needed for dispute votes to get confirmed/fail getting transmitted.",
+				))?,
+				registry,
+			)?,
+		};
+		Ok(Metrics(Some(metrics)))
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/batches/batch.rs

@@ -0,0 +1,209 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{collections::HashMap, time::Instant};
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{
+	request_response::{incoming::OutgoingResponseSender, v1::DisputeRequest},
+	PeerId,
+};
+use pezkuwi_node_primitives::SignedDisputeStatement;
+use pezkuwi_primitives::{CandidateReceiptV2 as CandidateReceipt, ValidatorIndex};
+
+use crate::receiver::{BATCH_COLLECTING_INTERVAL, MIN_KEEP_BATCH_ALIVE_VOTES};
+
+use super::MAX_BATCH_LIFETIME;
+
+/// A batch of votes to be imported into the `dispute-coordinator`.
+///
+/// Vote imports are way more efficient when performed in batches, hence we batch together incoming
+/// votes until the rate of incoming votes falls below a threshold, then we import into the dispute
+/// coordinator.
+///
+/// A `Batch` keeps track of the votes to be imported and the current incoming rate, on rate update
+/// it will "flush" in case the incoming rate dropped too low, preparing the import.
+pub struct Batch {
+	/// The actual candidate this batch is concerned with.
+	candidate_receipt: CandidateReceipt,
+
+	/// Cache of `CandidateHash` (candidate_receipt.hash()).
+	candidate_hash: CandidateHash,
+
+	/// All valid votes received in this batch so far.
+	///
+	/// We differentiate between valid and invalid votes, so we can detect (and drop) duplicates,
+	/// while still allowing validators to equivocate.
+	///
+	/// Detecting and rejecting duplicates is crucial in order to effectively enforce
+	/// `MIN_KEEP_BATCH_ALIVE_VOTES` per `BATCH_COLLECTING_INTERVAL`. If we would count duplicates
+	/// here, the mechanism would be broken.
+	valid_votes: HashMap<ValidatorIndex, SignedDisputeStatement>,
+
+	/// All invalid votes received in this batch so far.
+	invalid_votes: HashMap<ValidatorIndex, SignedDisputeStatement>,
+
+	/// How many votes have been batched since the last tick/creation.
+	votes_batched_since_last_tick: u32,
+
+	/// Expiry time for the batch.
+	///
+	/// By this time the latest this batch will get flushed.
+	best_before: Instant,
+
+	/// Requesters waiting for a response.
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+}
+
+/// Result of checking a batch every `BATCH_COLLECTING_INTERVAL`.
+pub(super) enum TickResult {
+	/// Batch is still alive, please call `tick` again at the given `Instant`.
+	Alive(Batch, Instant),
+	/// Batch is done, ready for import!
+	Done(PreparedImport),
+}
+
+/// Ready for import.
+pub struct PreparedImport {
+	pub candidate_receipt: CandidateReceipt,
+	pub statements: Vec<(SignedDisputeStatement, ValidatorIndex)>,
+	/// Information about original requesters.
+	pub requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+}
+
+impl From<Batch> for PreparedImport {
+	fn from(batch: Batch) -> Self {
+		let Batch {
+			candidate_receipt,
+			valid_votes,
+			invalid_votes,
+			requesters: pending_responses,
+			..
+		} = batch;
+
+		let statements = valid_votes
+			.into_iter()
+			.chain(invalid_votes.into_iter())
+			.map(|(index, statement)| (statement, index))
+			.collect();
+
+		Self { candidate_receipt, statements, requesters: pending_responses }
+	}
+}
+
+impl Batch {
+	/// Create a new empty batch based on the given `CandidateReceipt`.
+	///
+	/// To create a `Batch` use Batches::find_batch`.
+	///
+	/// Arguments:
+	///
+	/// * `candidate_receipt` - The candidate this batch is meant to track votes for.
+	/// * `now` - current time stamp for calculating the first tick.
+	///
+	/// Returns: A batch and the first `Instant` you are supposed to call `tick`.
+	pub(super) fn new(candidate_receipt: CandidateReceipt, now: Instant) -> (Self, Instant) {
+		let s = Self {
+			candidate_hash: candidate_receipt.hash(),
+			candidate_receipt,
+			valid_votes: HashMap::new(),
+			invalid_votes: HashMap::new(),
+			votes_batched_since_last_tick: 0,
+			best_before: Instant::now() + MAX_BATCH_LIFETIME,
+			requesters: Vec::new(),
+		};
+		let next_tick = s.calculate_next_tick(now);
+		(s, next_tick)
+	}
+
+	/// Receipt of the candidate this batch is batching votes for.
+	pub fn candidate_receipt(&self) -> &CandidateReceipt {
+		&self.candidate_receipt
+	}
+
+	/// Add votes from a validator into the batch.
+	///
+	/// The statements are supposed to be the valid and invalid statements received in a
+	/// `DisputeRequest`.
+	///
+	/// The given `pending_response` is the corresponding response sender for responding to `peer`.
+	/// If at least one of the votes is new as far as this batch is concerned we record the
+	/// pending_response, for later use. In case both votes are known already, we return the
+	/// response sender as an `Err` value.
+	pub fn add_votes(
+		&mut self,
+		valid_vote: (SignedDisputeStatement, ValidatorIndex),
+		invalid_vote: (SignedDisputeStatement, ValidatorIndex),
+		peer: PeerId,
+		pending_response: OutgoingResponseSender<DisputeRequest>,
+	) -> Result<(), OutgoingResponseSender<DisputeRequest>> {
+		debug_assert!(valid_vote.0.candidate_hash() == invalid_vote.0.candidate_hash());
+		debug_assert!(valid_vote.0.candidate_hash() == &self.candidate_hash);
+
+		let mut duplicate = true;
+
+		if self.valid_votes.insert(valid_vote.1, valid_vote.0).is_none() {
+			self.votes_batched_since_last_tick += 1;
+			duplicate = false;
+		}
+		if self.invalid_votes.insert(invalid_vote.1, invalid_vote.0).is_none() {
+			self.votes_batched_since_last_tick += 1;
+			duplicate = false;
+		}
+
+		if duplicate {
+			Err(pending_response)
+		} else {
+			self.requesters.push((peer, pending_response));
+			Ok(())
+		}
+	}
+
+	/// Check batch for liveness.
+	///
+	/// This function is supposed to be called at instants given at construction and as returned as
+	/// part of `TickResult`.
+	pub(super) fn tick(mut self, now: Instant) -> TickResult {
+		if self.votes_batched_since_last_tick >= MIN_KEEP_BATCH_ALIVE_VOTES &&
+			now < self.best_before
+		{
+			// Still good:
+			let next_tick = self.calculate_next_tick(now);
+			// Reset counter:
+			self.votes_batched_since_last_tick = 0;
+			TickResult::Alive(self, next_tick)
+		} else {
+			TickResult::Done(PreparedImport::from(self))
+		}
+	}
+
+	/// Calculate when the next tick should happen.
+	///
+	/// This will usually return `now + BATCH_COLLECTING_INTERVAL`, except if the lifetime of this
+	/// batch would exceed `MAX_BATCH_LIFETIME`.
+	///
+	/// # Arguments
+	///
+	/// * `now` - The current time.
+	fn calculate_next_tick(&self, now: Instant) -> Instant {
+		let next_tick = now + BATCH_COLLECTING_INTERVAL;
+		if next_tick < self.best_before {
+			next_tick
+		} else {
+			self.best_before
+		}
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/batches/mod.rs

@@ -0,0 +1,170 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{hash_map, HashMap},
+	time::{Duration, Instant},
+};
+
+use futures::future::pending;
+
+use pezkuwi_node_network_protocol::request_response::DISPUTE_REQUEST_TIMEOUT;
+use pezkuwi_primitives::{CandidateHash, CandidateReceiptV2 as CandidateReceipt};
+
+use crate::{
+	receiver::batches::{batch::TickResult, waiting_queue::PendingWake},
+	LOG_TARGET,
+};
+
+pub use self::batch::{Batch, PreparedImport};
+use self::waiting_queue::WaitingQueue;
+
+use super::{
+	error::{JfyiError, JfyiResult},
+	BATCH_COLLECTING_INTERVAL,
+};
+
+/// A single batch (per candidate) as managed by `Batches`.
+mod batch;
+
+/// Queue events in time and wait for them to become ready.
+mod waiting_queue;
+
+/// Safe-guard in case votes trickle in real slow.
+///
+/// If the batch life time exceeded the time the sender is willing to wait for a confirmation, we
+/// would trigger pointless re-sends.
+const MAX_BATCH_LIFETIME: Duration = DISPUTE_REQUEST_TIMEOUT.saturating_sub(Duration::from_secs(2));
+
+/// Limit the number of batches that can be alive at any given time.
+///
+/// Reasoning for this number, see guide.
+pub const MAX_BATCHES: usize = 1000;
+
+/// Manage batches.
+///
+/// - Batches can be found via `find_batch()` in order to add votes to them/check they exist.
+/// - Batches can be checked for being ready for flushing in order to import contained votes.
+pub struct Batches {
+	/// The batches we manage.
+	///
+	/// Kept invariants:
+	/// For each entry in `batches`, there exists an entry in `waiting_queue` as well - we wait on
+	/// all batches!
+	batches: HashMap<CandidateHash, Batch>,
+	/// Waiting queue for waiting for batches to become ready for `tick`.
+	///
+	/// Kept invariants by `Batches`:
+	/// For each entry in the `waiting_queue` there exists a corresponding entry in `batches`.
+	waiting_queue: WaitingQueue<CandidateHash>,
+}
+
+/// A found batch is either really found or got created so it can be found.
+pub enum FoundBatch<'a> {
+	/// Batch just got created.
+	Created(&'a mut Batch),
+	/// Batch already existed.
+	Found(&'a mut Batch),
+}
+
+impl Batches {
+	/// Create new empty `Batches`.
+	pub fn new() -> Self {
+		debug_assert!(
+			MAX_BATCH_LIFETIME > BATCH_COLLECTING_INTERVAL,
+			"Unexpectedly low `MAX_BATCH_LIFETIME`, please check parameters."
+		);
+		Self { batches: HashMap::new(), waiting_queue: WaitingQueue::new() }
+	}
+
+	/// Find a particular batch.
+	///
+	/// That is either find it, or we create it as reflected by the result `FoundBatch`.
+	pub fn find_batch(
+		&mut self,
+		candidate_hash: CandidateHash,
+		candidate_receipt: CandidateReceipt,
+	) -> JfyiResult<FoundBatch<'_>> {
+		if self.batches.len() >= MAX_BATCHES {
+			return Err(JfyiError::MaxBatchLimitReached);
+		}
+		debug_assert!(candidate_hash == candidate_receipt.hash());
+		let result = match self.batches.entry(candidate_hash) {
+			hash_map::Entry::Vacant(vacant) => {
+				let now = Instant::now();
+				let (created, ready_at) = Batch::new(candidate_receipt, now);
+				let pending_wake = PendingWake { payload: candidate_hash, ready_at };
+				self.waiting_queue.push(pending_wake);
+				FoundBatch::Created(vacant.insert(created))
+			},
+			hash_map::Entry::Occupied(occupied) => FoundBatch::Found(occupied.into_mut()),
+		};
+		Ok(result)
+	}
+
+	/// Wait for the next `tick` to check for ready batches.
+	///
+	/// This function blocks (returns `Poll::Pending`) until at least one batch can be
+	/// checked for readiness meaning that `BATCH_COLLECTING_INTERVAL` has passed since the last
+	/// check for that batch or it reached end of life.
+	///
+	/// If this `Batches` instance is empty (does not actually contain any batches), then this
+	/// function will always return `Poll::Pending`.
+	///
+	/// Returns: A `Vec` of all `PreparedImport`s from batches that became ready.
+	pub async fn check_batches(&mut self) -> Vec<PreparedImport> {
+		let now = Instant::now();
+
+		let mut imports = Vec::new();
+
+		// Wait for at least one batch to become ready:
+		self.waiting_queue.wait_ready(now).await;
+
+		// Process all ready entries:
+		while let Some(wake) = self.waiting_queue.pop_ready(now) {
+			let batch = self.batches.remove(&wake.payload);
+			debug_assert!(
+				batch.is_some(),
+				"Entries referenced in `waiting_queue` are supposed to exist!"
+			);
+			let batch = match batch {
+				None => return pending().await,
+				Some(batch) => batch,
+			};
+			match batch.tick(now) {
+				TickResult::Done(import) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?wake.payload,
+						"Batch became ready."
+					);
+					imports.push(import);
+				},
+				TickResult::Alive(old_batch, next_tick) => {
+					gum::trace!(
+						target: LOG_TARGET,
+						candidate_hash = ?wake.payload,
+						"Batch found to be still alive on check."
+					);
+					let pending_wake = PendingWake { payload: wake.payload, ready_at: next_tick };
+					self.waiting_queue.push(pending_wake);
+					self.batches.insert(wake.payload, old_batch);
+				},
+			}
+		}
+		imports
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/batches/waiting_queue.rs

@@ -0,0 +1,204 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{cmp::Ordering, collections::BinaryHeap, time::Instant};
+
+use futures::future::pending;
+use futures_timer::Delay;
+
+/// Wait asynchronously for given `Instant`s one after the other.
+///
+/// `PendingWake`s can be inserted and `WaitingQueue` makes `wait_ready()` to always wait for the
+/// next `Instant` in the queue.
+pub struct WaitingQueue<Payload> {
+	/// All pending wakes we are supposed to wait on in order.
+	pending_wakes: BinaryHeap<PendingWake<Payload>>,
+	/// Wait for next `PendingWake`.
+	timer: Option<Delay>,
+}
+
+/// Represents some event waiting to be processed at `ready_at`.
+///
+/// This is an event in `WaitingQueue`. It provides an `Ord` instance, that sorts descending with
+/// regard to `Instant` (so we get a `min-heap` with the earliest `Instant` at the top).
+#[derive(Eq, PartialEq)]
+pub struct PendingWake<Payload> {
+	pub payload: Payload,
+	pub ready_at: Instant,
+}
+
+impl<Payload: Eq + Ord> WaitingQueue<Payload> {
+	/// Get a new empty `WaitingQueue`.
+	///
+	/// If you call `pop` on this queue immediately, it will always return `Poll::Pending`.
+	pub fn new() -> Self {
+		Self { pending_wakes: BinaryHeap::new(), timer: None }
+	}
+
+	/// Push a `PendingWake`.
+	///
+	/// The next call to `wait_ready` will make sure to wake soon enough to process that new event
+	/// in a timely manner.
+	pub fn push(&mut self, wake: PendingWake<Payload>) {
+		self.pending_wakes.push(wake);
+		// Reset timer as it is potentially obsolete now:
+		self.timer = None;
+	}
+
+	/// Pop the next ready item.
+	///
+	/// This function does not wait, if nothing is ready right now as determined by the passed
+	/// `now` time stamp, this function simply returns `None`.
+	pub fn pop_ready(&mut self, now: Instant) -> Option<PendingWake<Payload>> {
+		let is_ready = self.pending_wakes.peek().map_or(false, |p| p.ready_at <= now);
+		if is_ready {
+			Some(self.pending_wakes.pop().expect("We just peeked. qed."))
+		} else {
+			None
+		}
+	}
+
+	/// Don't pop, just wait until something is ready.
+	///
+	/// Once this function returns `Poll::Ready(())` `pop_ready()` will return `Some`, if passed
+	/// the same `Instant`.
+	///
+	/// Whether ready or not is determined based on the passed time stamp `now` which should be the
+	/// current time as returned by `Instant::now()`
+	///
+	/// This function waits asynchronously for an item to become ready. If there is no more item,
+	/// this call will wait forever (return Poll::Pending without scheduling a wake).
+	pub async fn wait_ready(&mut self, now: Instant) {
+		if let Some(timer) = &mut self.timer {
+			// Previous timer was not done yet.
+			timer.await
+		}
+
+		let next_waiting = self.pending_wakes.peek();
+		let is_ready = next_waiting.map_or(false, |p| p.ready_at <= now);
+		if is_ready {
+			return;
+		}
+
+		self.timer = next_waiting.map(|p| Delay::new(p.ready_at.duration_since(now)));
+		match &mut self.timer {
+			None => return pending().await,
+			Some(timer) => timer.await,
+		}
+	}
+}
+
+impl<Payload: Eq + Ord> PartialOrd<PendingWake<Payload>> for PendingWake<Payload> {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl<Payload: Ord> Ord for PendingWake<Payload> {
+	fn cmp(&self, other: &Self) -> Ordering {
+		// Reverse order for min-heap:
+		match other.ready_at.cmp(&self.ready_at) {
+			Ordering::Equal => other.payload.cmp(&self.payload),
+			o => o,
+		}
+	}
+}
+#[cfg(test)]
+mod tests {
+	use std::{
+		task::Poll,
+		time::{Duration, Instant},
+	};
+
+	use assert_matches::assert_matches;
+	use futures::{future::poll_fn, pin_mut, Future};
+
+	use crate::LOG_TARGET;
+
+	use super::{PendingWake, WaitingQueue};
+
+	#[test]
+	fn wait_ready_waits_for_earliest_event_always() {
+		sp_tracing::try_init_simple();
+		let mut queue = WaitingQueue::new();
+		let now = Instant::now();
+		let start = now;
+		queue.push(PendingWake { payload: 1u32, ready_at: now + Duration::from_millis(3) });
+		// Push another one in order:
+		queue.push(PendingWake { payload: 2u32, ready_at: now + Duration::from_millis(5) });
+		// Push one out of order:
+		queue.push(PendingWake { payload: 0u32, ready_at: now + Duration::from_millis(1) });
+		// Push another one at same timestamp (should become ready at the same time)
+		queue.push(PendingWake { payload: 10u32, ready_at: now + Duration::from_millis(1) });
+
+		futures::executor::block_on(async move {
+			// No time passed yet - nothing should be ready.
+			assert!(queue.pop_ready(now).is_none(), "No time has passed, nothing should be ready");
+
+			// Receive them in order at expected times:
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After first wait.");
+
+			let now = start + Duration::from_millis(1);
+			assert!(Instant::now() - start >= Duration::from_millis(1));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(0u32));
+			// One more should be ready:
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(10u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After second wait.");
+			let now = start + Duration::from_millis(3);
+			assert!(Instant::now() - start >= Duration::from_millis(3));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(1u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			// Push in between wait:
+			poll_fn(|cx| {
+				let fut = queue.wait_ready(now);
+				pin_mut!(fut);
+				assert_matches!(fut.poll(cx), Poll::Pending);
+				Poll::Ready(())
+			})
+			.await;
+			queue.push(PendingWake { payload: 3u32, ready_at: start + Duration::from_millis(4) });
+
+			queue.wait_ready(now).await;
+			// Newly pushed element should have become ready:
+			gum::trace!(target: LOG_TARGET, "After third wait.");
+			let now = start + Duration::from_millis(4);
+			assert!(Instant::now() - start >= Duration::from_millis(4));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(3u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			queue.wait_ready(now).await;
+			gum::trace!(target: LOG_TARGET, "After fourth wait.");
+			let now = start + Duration::from_millis(5);
+			assert!(Instant::now() - start >= Duration::from_millis(5));
+			assert_eq!(queue.pop_ready(now).map(|p| p.payload), Some(2u32));
+			assert!(queue.pop_ready(now).is_none(), "No more entry expected to be ready.");
+
+			// queue empty - should wait forever now:
+			poll_fn(|cx| {
+				let fut = queue.wait_ready(now);
+				pin_mut!(fut);
+				assert_matches!(fut.poll(cx), Poll::Pending);
+				Poll::Ready(())
+			})
+			.await;
+		});
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/error.rs

@@ -0,0 +1,97 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Error handling related code and Error/Result definitions.
+
+use fatality::Nested;
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{request_response::incoming, PeerId};
+use pezkuwi_node_subsystem_util::runtime;
+use pezkuwi_primitives::AuthorityDiscoveryId;
+
+use crate::LOG_TARGET;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	#[fatal(forward)]
+	#[error("Retrieving next incoming request failed.")]
+	IncomingRequest(#[from] incoming::Error),
+
+	#[error("Sending back response to peers {0:#?} failed.")]
+	SendResponses(Vec<PeerId>),
+
+	#[error("Changing peer's ({0}) reputation failed.")]
+	SetPeerReputation(PeerId),
+
+	#[error("Dispute request with invalid signatures, from peer {0}.")]
+	InvalidSignature(PeerId),
+
+	#[error("Received votes from peer {0} have been completely redundant.")]
+	RedundantMessage(PeerId),
+
+	#[error("Import of dispute got canceled for candidate {0} - import failed for some reason.")]
+	ImportCanceled(CandidateHash),
+
+	#[error("Peer {0} attempted to participate in dispute and is not a validator.")]
+	NotAValidator(PeerId),
+
+	#[error("Force flush for batch that could not be found attempted, candidate hash: {0}")]
+	ForceFlushBatchDoesNotExist(CandidateHash),
+
+	// Should never happen in practice:
+	#[error("We needed to drop messages, because we reached limit on concurrent batches.")]
+	MaxBatchLimitReached,
+
+	#[error("Authority {0} sent messages at a too high rate.")]
+	AuthorityFlooding(AuthorityDiscoveryId),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub type JfyiResult<T> = std::result::Result<T, JfyiError>;
+
+/// Utility for eating top level errors and log them.
+///
+/// We basically always want to try and continue on error. This utility function is meant to
+/// consume top-level errors by simply logging them.
+pub fn log_error(result: Result<()>) -> std::result::Result<(), FatalError> {
+	match result.into_nested()? {
+		Err(error @ JfyiError::ImportCanceled(_)) => {
+			gum::debug!(target: LOG_TARGET, error = ?error);
+			Ok(())
+		},
+		Err(JfyiError::NotAValidator(peer)) => {
+			gum::debug!(
+				target: LOG_TARGET,
+				?peer,
+				"Dropping message from peer (unknown authority id)"
+			);
+			Ok(())
+		},
+		Err(error) => {
+			gum::warn!(target: LOG_TARGET, error = ?error);
+			Ok(())
+		},
+		Ok(()) => Ok(()),
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/mod.rs

@@ -0,0 +1,522 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	pin::Pin,
+	task::{Context, Poll},
+	time::Duration,
+};
+
+use futures::{
+	channel::oneshot,
+	future::poll_fn,
+	pin_mut,
+	stream::{FuturesUnordered, StreamExt},
+	Future,
+};
+
+use gum::CandidateHash;
+use pezkuwi_node_network_protocol::{
+	authority_discovery::AuthorityDiscovery,
+	request_response::{
+		incoming::{self, OutgoingResponse, OutgoingResponseSender},
+		v1::{DisputeRequest, DisputeResponse},
+		IncomingRequest, IncomingRequestReceiver,
+	},
+	PeerId, UnifiedReputationChange as Rep,
+};
+use pezkuwi_node_primitives::DISPUTE_WINDOW;
+use pezkuwi_node_subsystem::{
+	messages::{DisputeCoordinatorMessage, ImportStatementsResult},
+	overseer,
+};
+use pezkuwi_node_subsystem_util::{runtime, runtime::RuntimeInfo};
+
+use crate::{
+	metrics::{FAILED, SUCCEEDED},
+	Metrics, LOG_TARGET,
+};
+
+mod error;
+
+/// Rate limiting queues for incoming requests by peers.
+mod peer_queues;
+
+/// Batch imports together.
+mod batches;
+
+use self::{
+	batches::{Batches, FoundBatch, PreparedImport},
+	error::{log_error, JfyiError, JfyiResult, Result},
+	peer_queues::PeerQueues,
+};
+
+const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Received message could not be decoded.");
+const COST_INVALID_SIGNATURE: Rep = Rep::Malicious("Signatures were invalid.");
+const COST_NOT_A_VALIDATOR: Rep = Rep::CostMajor("Reporting peer was not a validator.");
+
+/// Invalid imports can be caused by flooding, e.g. by a disabled validator.
+const COST_INVALID_IMPORT: Rep =
+	Rep::CostMinor("Import was deemed invalid by dispute-coordinator.");
+
+/// How many votes must have arrived in the last `BATCH_COLLECTING_INTERVAL`
+///
+/// in order for a batch to stay alive and not get flushed/imported to the dispute-coordinator.
+///
+/// This ensures a timely import of batches.
+#[cfg(not(test))]
+pub const MIN_KEEP_BATCH_ALIVE_VOTES: u32 = 10;
+#[cfg(test)]
+pub const MIN_KEEP_BATCH_ALIVE_VOTES: u32 = 2;
+
+/// Time we allow to pass for new votes to trickle in.
+///
+/// See `MIN_KEEP_BATCH_ALIVE_VOTES` above.
+/// Should be greater or equal to `RECEIVE_RATE_LIMIT` (there is no point in checking any faster).
+pub const BATCH_COLLECTING_INTERVAL: Duration = Duration::from_millis(500);
+
+/// State for handling incoming `DisputeRequest` messages.
+pub struct DisputesReceiver<Sender, AD> {
+	/// Access to session information.
+	runtime: RuntimeInfo,
+
+	/// Subsystem sender for communication with other subsystems.
+	sender: Sender,
+
+	/// Channel to retrieve incoming requests from.
+	receiver: IncomingRequestReceiver<DisputeRequest>,
+
+	/// Rate limiting queue for each peer (only authorities).
+	peer_queues: PeerQueues,
+
+	/// Currently active batches of imports per candidate.
+	batches: Batches,
+
+	/// Authority discovery service:
+	authority_discovery: AD,
+
+	/// Imports currently being processed by the `dispute-coordinator`.
+	pending_imports: FuturesUnordered<PendingImport>,
+
+	/// Log received requests.
+	metrics: Metrics,
+}
+
+/// Messages as handled by this receiver internally.
+enum MuxedMessage {
+	/// An import got confirmed by the coordinator.
+	///
+	/// We need to handle those for two reasons:
+	///
+	/// - We need to make sure responses are actually sent (therefore we need to await futures
+	/// promptly).
+	/// - We need to punish peers whose import got rejected.
+	ConfirmedImport(ImportResult),
+
+	/// A new request has arrived and should be handled.
+	NewRequest(IncomingRequest<DisputeRequest>),
+
+	/// Rate limit timer hit - is time to process one row of messages.
+	///
+	/// This is the result of calling `self.peer_queues.pop_reqs()`.
+	WakePeerQueuesPopReqs(Vec<IncomingRequest<DisputeRequest>>),
+
+	/// It is time to check batches.
+	///
+	/// Every `BATCH_COLLECTING_INTERVAL` we check whether less than `MIN_KEEP_BATCH_ALIVE_VOTES`
+	/// new votes arrived, if so the batch is ready for import.
+	///
+	/// This is the result of calling `self.batches.check_batches()`.
+	WakeCheckBatches(Vec<PreparedImport>),
+}
+
+impl<Sender, AD> DisputesReceiver<Sender, AD>
+where
+	AD: AuthorityDiscovery,
+	Sender: overseer::DisputeDistributionSenderTrait,
+{
+	/// Create a new receiver which can be `run`.
+	pub fn new(
+		sender: Sender,
+		receiver: IncomingRequestReceiver<DisputeRequest>,
+		authority_discovery: AD,
+		metrics: Metrics,
+	) -> Self {
+		let runtime = RuntimeInfo::new_with_config(runtime::Config {
+			keystore: None,
+			session_cache_lru_size: DISPUTE_WINDOW.get(),
+		});
+		Self {
+			runtime,
+			sender,
+			receiver,
+			peer_queues: PeerQueues::new(),
+			batches: Batches::new(),
+			authority_discovery,
+			pending_imports: FuturesUnordered::new(),
+			metrics,
+		}
+	}
+
+	/// Get that receiver started.
+	///
+	/// This is an endless loop and should be spawned into its own task.
+	pub async fn run(mut self) {
+		loop {
+			match log_error(self.run_inner().await) {
+				Ok(()) => {},
+				Err(fatal) => {
+					gum::debug!(
+						target: LOG_TARGET,
+						error = ?fatal,
+						"Shutting down"
+					);
+					return;
+				},
+			}
+		}
+	}
+
+	/// Actual work happening here in three phases:
+	///
+	/// 1. Receive and queue incoming messages until the rate limit timer hits.
+	/// 2. Do import/batching for the head of all queues.
+	/// 3. Check and flush any ready batches.
+	async fn run_inner(&mut self) -> Result<()> {
+		let msg = self.receive_message().await?;
+
+		match msg {
+			MuxedMessage::NewRequest(req) => {
+				// Phase 1:
+				self.metrics.on_received_request();
+				self.dispatch_to_queues(req).await?;
+			},
+			MuxedMessage::WakePeerQueuesPopReqs(reqs) => {
+				// Phase 2:
+				for req in reqs {
+					// No early return - we cannot cancel imports of one peer, because the import of
+					// another failed:
+					match log_error(self.start_import_or_batch(req).await) {
+						Ok(()) => {},
+						Err(fatal) => return Err(fatal.into()),
+					}
+				}
+			},
+			MuxedMessage::WakeCheckBatches(ready_imports) => {
+				// Phase 3:
+				self.import_ready_batches(ready_imports).await;
+			},
+			MuxedMessage::ConfirmedImport(import_result) => {
+				self.update_imported_requests_metrics(&import_result);
+				// Confirm imports to requesters/punish them on invalid imports:
+				send_responses_to_requesters(import_result).await?;
+			},
+		}
+
+		Ok(())
+	}
+
+	/// Receive one `MuxedMessage`.
+	///
+	///
+	/// Dispatching events to messages as they happen.
+	async fn receive_message(&mut self) -> Result<MuxedMessage> {
+		poll_fn(|ctx| {
+			// In case of Ready(None), we want to wait for pending requests:
+			if let Poll::Ready(Some(v)) = self.pending_imports.poll_next_unpin(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::ConfirmedImport(v?)));
+			}
+
+			let rate_limited = self.peer_queues.pop_reqs();
+			pin_mut!(rate_limited);
+			// We poll rate_limit before batches, so we don't unnecessarily delay importing to
+			// batches.
+			if let Poll::Ready(reqs) = rate_limited.poll(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::WakePeerQueuesPopReqs(reqs)));
+			}
+
+			let ready_batches = self.batches.check_batches();
+			pin_mut!(ready_batches);
+			if let Poll::Ready(ready_batches) = ready_batches.poll(ctx) {
+				return Poll::Ready(Ok(MuxedMessage::WakeCheckBatches(ready_batches)));
+			}
+
+			let next_req = self.receiver.recv(|| vec![COST_INVALID_REQUEST]);
+			pin_mut!(next_req);
+			if let Poll::Ready(r) = next_req.poll(ctx) {
+				return match r {
+					Err(e) => Poll::Ready(Err(incoming::Error::from(e).into())),
+					Ok(v) => Poll::Ready(Ok(MuxedMessage::NewRequest(v))),
+				};
+			}
+			Poll::Pending
+		})
+		.await
+	}
+
+	/// Process incoming requests.
+	///
+	/// - Check sender is authority
+	/// - Dispatch message to corresponding queue in `peer_queues`.
+	/// - If queue is full, drop message and change reputation of sender.
+	async fn dispatch_to_queues(&mut self, req: IncomingRequest<DisputeRequest>) -> JfyiResult<()> {
+		let peer = req.peer;
+		// Only accept messages from validators, in case there are multiple `AuthorityId`s, we
+		// just take the first one. On session boundaries this might allow validators to double
+		// their rate limit for a short period of time, which seems acceptable.
+		let authority_id = match self
+			.authority_discovery
+			.get_authority_ids_by_peer_id(peer)
+			.await
+			.and_then(|s| s.into_iter().next())
+		{
+			None => {
+				req.send_outgoing_response(OutgoingResponse {
+					result: Err(()),
+					reputation_changes: vec![COST_NOT_A_VALIDATOR],
+					sent_feedback: None,
+				})
+				.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+				return Err(JfyiError::NotAValidator(peer).into());
+			},
+			Some(auth_id) => auth_id,
+		};
+
+		// Queue request:
+		if let Err((authority_id, req)) = self.peer_queues.push_req(authority_id, req) {
+			gum::debug!(
+				target: LOG_TARGET,
+				?authority_id,
+				?peer,
+				"Peer hit the rate limit - dropping message."
+			);
+			req.send_outgoing_response(OutgoingResponse {
+				result: Err(()),
+				reputation_changes: vec![],
+				sent_feedback: None,
+			})
+			.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+			return Err(JfyiError::AuthorityFlooding(authority_id));
+		}
+		Ok(())
+	}
+
+	/// Start importing votes for the given request or batch.
+	///
+	/// Signature check and in case we already have an existing batch we import to that batch,
+	/// otherwise import to `dispute-coordinator` directly and open a batch.
+	async fn start_import_or_batch(
+		&mut self,
+		incoming: IncomingRequest<DisputeRequest>,
+	) -> Result<()> {
+		let IncomingRequest { peer, payload, pending_response } = incoming;
+
+		let info = self
+			.runtime
+			.get_session_info_by_index(
+				&mut self.sender,
+				payload.0.candidate_receipt.descriptor.relay_parent(),
+				payload.0.session_index,
+			)
+			.await?;
+
+		let votes_result = payload.0.try_into_signed_votes(&info.session_info);
+
+		let (candidate_receipt, valid_vote, invalid_vote) = match votes_result {
+			Err(()) => {
+				// Signature invalid:
+				pending_response
+					.send_outgoing_response(OutgoingResponse {
+						result: Err(()),
+						reputation_changes: vec![COST_INVALID_SIGNATURE],
+						sent_feedback: None,
+					})
+					.map_err(|_| JfyiError::SetPeerReputation(peer))?;
+
+				return Err(From::from(JfyiError::InvalidSignature(peer)));
+			},
+			Ok(votes) => votes,
+		};
+
+		let candidate_hash = *valid_vote.0.candidate_hash();
+
+		match self.batches.find_batch(candidate_hash, candidate_receipt)? {
+			FoundBatch::Created(batch) => {
+				// There was no entry yet - start import immediately:
+				gum::trace!(
+					target: LOG_TARGET,
+					?candidate_hash,
+					?peer,
+					"No batch yet - triggering immediate import"
+				);
+				let import = PreparedImport {
+					candidate_receipt: batch.candidate_receipt().clone(),
+					statements: vec![valid_vote, invalid_vote],
+					requesters: vec![(peer, pending_response)],
+				};
+				self.start_import(import).await;
+			},
+			FoundBatch::Found(batch) => {
+				gum::trace!(target: LOG_TARGET, ?candidate_hash, "Batch exists - batching request");
+				let batch_result =
+					batch.add_votes(valid_vote, invalid_vote, peer, pending_response);
+
+				if let Err(pending_response) = batch_result {
+					// We don't expect honest peers to send redundant votes within a single batch,
+					// as the timeout for retry is much higher. Still we don't want to punish the
+					// node as it might not be the node's fault. Some other (malicious) node could
+					// have been faster sending the same votes in order to harm the reputation of
+					// that honest node. Given that we already have a rate limit, if a validator
+					// chooses to waste available rate with redundant votes - so be it. The actual
+					// dispute resolution is unaffected.
+					gum::debug!(
+						target: LOG_TARGET,
+						?peer,
+						"Peer sent completely redundant votes within a single batch - that looks fishy!",
+					);
+					pending_response
+						.send_outgoing_response(OutgoingResponse {
+							// While we have seen duplicate votes, we cannot confirm as we don't
+							// know yet whether the batch is going to be confirmed, so we assume
+							// the worst. We don't want to push the pending response to the batch
+							// either as that would be unbounded, only limited by the rate limit.
+							result: Err(()),
+							reputation_changes: Vec::new(),
+							sent_feedback: None,
+						})
+						.map_err(|_| JfyiError::SendResponses(vec![peer]))?;
+					return Err(From::from(JfyiError::RedundantMessage(peer)));
+				}
+			},
+		}
+
+		Ok(())
+	}
+
+	/// Trigger import into the dispute-coordinator of ready batches (`PreparedImport`s).
+	async fn import_ready_batches(&mut self, ready_imports: Vec<PreparedImport>) {
+		for import in ready_imports {
+			self.start_import(import).await;
+		}
+	}
+
+	/// Start import and add response receiver to `pending_imports`.
+	async fn start_import(&mut self, import: PreparedImport) {
+		let PreparedImport { candidate_receipt, statements, requesters } = import;
+		let (session_index, candidate_hash) = match statements.iter().next() {
+			None => {
+				gum::debug!(
+					target: LOG_TARGET,
+					candidate_hash = ?candidate_receipt.hash(),
+					"Not importing empty batch"
+				);
+				return;
+			},
+			Some(vote) => (vote.0.session_index(), *vote.0.candidate_hash()),
+		};
+
+		let (pending_confirmation, confirmation_rx) = oneshot::channel();
+		self.sender
+			.send_message(DisputeCoordinatorMessage::ImportStatements {
+				candidate_receipt,
+				session: session_index,
+				statements,
+				pending_confirmation: Some(pending_confirmation),
+			})
+			.await;
+
+		let pending =
+			PendingImport { candidate_hash, requesters, pending_response: confirmation_rx };
+
+		self.pending_imports.push(pending);
+	}
+
+	fn update_imported_requests_metrics(&self, result: &ImportResult) {
+		let label = match result.result {
+			ImportStatementsResult::ValidImport => SUCCEEDED,
+			ImportStatementsResult::InvalidImport => FAILED,
+		};
+		self.metrics.on_imported(label, result.requesters.len());
+	}
+}
+
+async fn send_responses_to_requesters(import_result: ImportResult) -> JfyiResult<()> {
+	let ImportResult { requesters, result } = import_result;
+
+	let mk_response = match result {
+		ImportStatementsResult::ValidImport => || OutgoingResponse {
+			result: Ok(DisputeResponse::Confirmed),
+			reputation_changes: Vec::new(),
+			sent_feedback: None,
+		},
+		ImportStatementsResult::InvalidImport => || OutgoingResponse {
+			result: Err(()),
+			reputation_changes: vec![COST_INVALID_IMPORT],
+			sent_feedback: None,
+		},
+	};
+
+	let mut sending_failed_for = Vec::new();
+	for (peer, pending_response) in requesters {
+		if let Err(()) = pending_response.send_outgoing_response(mk_response()) {
+			sending_failed_for.push(peer);
+		}
+	}
+
+	if !sending_failed_for.is_empty() {
+		Err(JfyiError::SendResponses(sending_failed_for))
+	} else {
+		Ok(())
+	}
+}
+
+/// A future that resolves into an `ImportResult` when ready.
+///
+/// This future is used on `dispute-coordinator` import messages for the oneshot response receiver
+/// to:
+/// - Keep track of concerned `CandidateHash` for reporting errors.
+/// - Keep track of requesting peers so we can confirm the import/punish them on invalid imports.
+struct PendingImport {
+	candidate_hash: CandidateHash,
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+	pending_response: oneshot::Receiver<ImportStatementsResult>,
+}
+
+/// A `PendingImport` becomes an `ImportResult` once done.
+struct ImportResult {
+	/// Requesters of that import.
+	requesters: Vec<(PeerId, OutgoingResponseSender<DisputeRequest>)>,
+	/// Actual result of the import.
+	result: ImportStatementsResult,
+}
+
+impl PendingImport {
+	async fn wait_for_result(&mut self) -> JfyiResult<ImportResult> {
+		let result = (&mut self.pending_response)
+			.await
+			.map_err(|_| JfyiError::ImportCanceled(self.candidate_hash))?;
+		Ok(ImportResult { requesters: std::mem::take(&mut self.requesters), result })
+	}
+}
+
+impl Future for PendingImport {
+	type Output = JfyiResult<ImportResult>;
+	fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+		let fut = self.wait_for_result();
+		pin_mut!(fut);
+		fut.poll(cx)
+	}
+}

pezkuwi/node/network/dispute-distribution/src/receiver/peer_queues.rs

@@ -0,0 +1,141 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{hash_map::Entry, HashMap, VecDeque};
+
+use futures::future::pending;
+use futures_timer::Delay;
+use pezkuwi_node_network_protocol::request_response::{v1::DisputeRequest, IncomingRequest};
+use pezkuwi_primitives::AuthorityDiscoveryId;
+
+use crate::RECEIVE_RATE_LIMIT;
+
+/// How many messages we are willing to queue per peer (validator).
+///
+/// The larger this value is, the larger bursts are allowed to be without us dropping messages. On
+/// the flip side this gets allocated per validator, so for a size of 10 this will result
+/// in `10_000 * size_of(IncomingRequest)` in the worst case.
+///
+/// `PEER_QUEUE_CAPACITY` must not be 0 for obvious reasons.
+#[cfg(not(test))]
+pub const PEER_QUEUE_CAPACITY: usize = 10;
+#[cfg(test)]
+pub const PEER_QUEUE_CAPACITY: usize = 2;
+
+/// Queues for messages from authority peers for rate limiting.
+///
+/// Invariants ensured:
+///
+/// 1. No queue will ever have more than `PEER_QUEUE_CAPACITY` elements.
+/// 2. There are no empty queues. Whenever a queue gets empty, it is removed. This way checking
+///    whether there are any messages queued is cheap.
+/// 3. As long as not empty, `pop_reqs` will, if called in sequence, not return `Ready` more often
+///    than once for every `RECEIVE_RATE_LIMIT`, but it will always return Ready eventually.
+/// 4. If empty `pop_reqs` will never return `Ready`, but will always be `Pending`.
+pub struct PeerQueues {
+	/// Actual queues.
+	queues: HashMap<AuthorityDiscoveryId, VecDeque<IncomingRequest<DisputeRequest>>>,
+
+	/// Delay timer for establishing the rate limit.
+	rate_limit_timer: Option<Delay>,
+}
+
+impl PeerQueues {
+	/// New empty `PeerQueues`.
+	pub fn new() -> Self {
+		Self { queues: HashMap::new(), rate_limit_timer: None }
+	}
+
+	/// Push an incoming request for a given authority.
+	///
+	/// Returns: `Ok(())` if succeeded, `Err((args))` if capacity is reached.
+	pub fn push_req(
+		&mut self,
+		peer: AuthorityDiscoveryId,
+		req: IncomingRequest<DisputeRequest>,
+	) -> Result<(), (AuthorityDiscoveryId, IncomingRequest<DisputeRequest>)> {
+		let queue = match self.queues.entry(peer) {
+			Entry::Vacant(vacant) => vacant.insert(VecDeque::new()),
+			Entry::Occupied(occupied) => {
+				if occupied.get().len() >= PEER_QUEUE_CAPACITY {
+					return Err((occupied.key().clone(), req));
+				}
+				occupied.into_mut()
+			},
+		};
+		queue.push_back(req);
+
+		// We have at least one element to process - rate limit `timer` needs to exist now:
+		self.ensure_timer();
+		Ok(())
+	}
+
+	/// Pop all heads and return them for processing.
+	///
+	/// This gets one message from each peer that has sent at least one.
+	///
+	/// This function is rate limited, if called in sequence it will not return more often than
+	/// every `RECEIVE_RATE_LIMIT`.
+	///
+	/// NOTE: If empty this function will not return `Ready` at all, but will always be `Pending`.
+	pub async fn pop_reqs(&mut self) -> Vec<IncomingRequest<DisputeRequest>> {
+		self.wait_for_timer().await;
+
+		let mut heads = Vec::with_capacity(self.queues.len());
+		let old_queues = std::mem::replace(&mut self.queues, HashMap::new());
+		for (k, mut queue) in old_queues.into_iter() {
+			let front = queue.pop_front();
+			debug_assert!(front.is_some(), "Invariant that queues are never empty is broken.");
+
+			if let Some(front) = front {
+				heads.push(front);
+			}
+			if !queue.is_empty() {
+				self.queues.insert(k, queue);
+			}
+		}
+
+		if !self.is_empty() {
+			// Still not empty - we should get woken at some point.
+			self.ensure_timer();
+		}
+
+		heads
+	}
+
+	/// Whether or not all queues are empty.
+	pub fn is_empty(&self) -> bool {
+		self.queues.is_empty()
+	}
+
+	/// Ensure there is an active `timer`.
+	///
+	/// Checks whether one exists and if not creates one.
+	fn ensure_timer(&mut self) -> &mut Delay {
+		self.rate_limit_timer.get_or_insert(Delay::new(RECEIVE_RATE_LIMIT))
+	}
+
+	/// Wait for `timer` if it exists, or be `Pending` forever.
+	///
+	/// Afterwards it gets set back to `None`.
+	async fn wait_for_timer(&mut self) {
+		match self.rate_limit_timer.as_mut() {
+			None => pending().await,
+			Some(timer) => timer.await,
+		}
+		self.rate_limit_timer = None;
+	}
+}

pezkuwi/node/network/dispute-distribution/src/sender/error.rs

@@ -0,0 +1,73 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Error handling related code and Error/Result definitions.
+
+use pezkuwi_node_primitives::disputes::DisputeMessageCheckError;
+use pezkuwi_node_subsystem::SubsystemError;
+use pezkuwi_node_subsystem_util::runtime;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal]
+	#[error("Spawning subsystem task failed")]
+	SpawnTask(#[source] SubsystemError),
+
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	/// We need available active heads for finding relevant authorities.
+	#[error("No active heads available - needed for finding relevant authorities.")]
+	NoActiveHeads,
+
+	/// This error likely indicates a bug in the coordinator.
+	#[error("Oneshot for asking dispute coordinator for active disputes got canceled.")]
+	AskActiveDisputesCanceled,
+
+	/// This error likely indicates a bug in the coordinator.
+	#[error("Oneshot for asking dispute coordinator for candidate votes got canceled.")]
+	AskCandidateVotesCanceled,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// We were not able to successfully construct a `DisputeMessage` from disputes votes.
+	#[error("Invalid dispute encountered")]
+	InvalidDisputeFromCoordinator(#[source] DisputeMessageCheckError),
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// We did not receive votes on both sides for `CandidateVotes` received from the coordinator.
+	#[error("Missing votes for valid dispute")]
+	MissingVotesFromCoordinator,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// `SignedDisputeStatement` could not be reconstructed from recorded statements.
+	#[error("Invalid statements from coordinator")]
+	InvalidStatementFromCoordinator,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// A statement's `ValidatorIndex` could not be looked up.
+	#[error("ValidatorIndex of statement could not be found")]
+	InvalidValidatorIndexFromCoordinator,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type JfyiErrorResult<T> = std::result::Result<T, JfyiError>;

pezkuwi/node/network/dispute-distribution/src/sender/mod.rs

@@ -0,0 +1,392 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{BTreeMap, HashMap, HashSet},
+	pin::Pin,
+	task::Poll,
+	time::Duration,
+};
+
+use futures::{channel::oneshot, future::poll_fn, Future};
+
+use futures_timer::Delay;
+use indexmap::{map::Entry, IndexMap};
+use pezkuwi_node_network_protocol::request_response::v1::DisputeRequest;
+use pezkuwi_node_primitives::{DisputeMessage, DisputeStatus};
+use pezkuwi_node_subsystem::{
+	messages::DisputeCoordinatorMessage, overseer, ActiveLeavesUpdate, SubsystemSender,
+};
+use pezkuwi_node_subsystem_util::{nesting_sender::NestingSender, runtime::RuntimeInfo};
+use pezkuwi_primitives::{CandidateHash, Hash, SessionIndex};
+
+/// For each ongoing dispute we have a `SendTask` which takes care of it.
+///
+/// It is going to spawn real tasks as it sees fit for getting the votes of the particular dispute
+/// out.
+///
+/// As we assume disputes have a priority, we start sending for disputes in the order
+/// `start_sender` got called.
+mod send_task;
+use send_task::SendTask;
+pub use send_task::TaskFinish;
+
+/// Error and [`Result`] type for sender.
+mod error;
+pub use error::{Error, FatalError, JfyiError, Result};
+
+use self::error::JfyiErrorResult;
+use crate::{Metrics, LOG_TARGET, SEND_RATE_LIMIT};
+
+/// Messages as sent by background tasks.
+#[derive(Debug)]
+pub enum DisputeSenderMessage {
+	/// A task finished.
+	TaskFinish(TaskFinish),
+	/// A request for active disputes to the dispute-coordinator finished.
+	ActiveDisputesReady(JfyiErrorResult<BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>>),
+}
+
+/// The `DisputeSender` keeps track of all ongoing disputes we need to send statements out.
+///
+/// For each dispute a `SendTask` is responsible for sending to the concerned validators for that
+/// particular dispute. The `DisputeSender` keeps track of those tasks, informs them about new
+/// sessions/validator sets and cleans them up when they become obsolete.
+///
+/// The unit of work for the  `DisputeSender` is a dispute, represented by `SendTask`s.
+pub struct DisputeSender<M> {
+	/// All heads we currently consider active.
+	active_heads: Vec<Hash>,
+
+	/// List of currently active sessions.
+	///
+	/// Value is the hash that was used for the query.
+	active_sessions: HashMap<SessionIndex, Hash>,
+
+	/// All ongoing dispute sending this subsystem is aware of.
+	///
+	/// Using an `IndexMap` so items can be iterated in the order of insertion.
+	disputes: IndexMap<CandidateHash, SendTask<M>>,
+
+	/// Sender to be cloned for `SendTask`s.
+	tx: NestingSender<M, DisputeSenderMessage>,
+
+	/// `Some` if we are waiting for a response `DisputeCoordinatorMessage::ActiveDisputes`.
+	waiting_for_active_disputes: Option<WaitForActiveDisputesState>,
+
+	/// Future for delaying too frequent creation of dispute sending tasks.
+	rate_limit: RateLimit,
+
+	/// Metrics for reporting stats about sent requests.
+	metrics: Metrics,
+}
+
+/// State we keep while waiting for active disputes.
+///
+/// When we send `DisputeCoordinatorMessage::ActiveDisputes`, this is the state we keep while
+/// waiting for the response.
+struct WaitForActiveDisputesState {
+	/// Have we seen any new sessions since last refresh?
+	have_new_sessions: bool,
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl<M: 'static + Send + Sync> DisputeSender<M> {
+	/// Create a new `DisputeSender` which can be used to start dispute sending.
+	pub fn new(tx: NestingSender<M, DisputeSenderMessage>, metrics: Metrics) -> Self {
+		Self {
+			active_heads: Vec::new(),
+			active_sessions: HashMap::new(),
+			disputes: IndexMap::new(),
+			tx,
+			waiting_for_active_disputes: None,
+			rate_limit: RateLimit::new(),
+			metrics,
+		}
+	}
+
+	/// Create a `SendTask` for a particular new dispute.
+	///
+	/// This function is rate-limited by `SEND_RATE_LIMIT`. It will block if called too frequently
+	/// in order to maintain the limit.
+	pub async fn start_sender<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		msg: DisputeMessage,
+	) -> Result<()> {
+		let req: DisputeRequest = msg.into();
+		let candidate_hash = req.0.candidate_receipt.hash();
+		match self.disputes.entry(candidate_hash) {
+			Entry::Occupied(_) => {
+				gum::trace!(target: LOG_TARGET, ?candidate_hash, "Dispute sending already active.");
+				return Ok(());
+			},
+			Entry::Vacant(vacant) => {
+				self.rate_limit.limit("in start_sender", candidate_hash).await;
+
+				let send_task = SendTask::new(
+					ctx,
+					runtime,
+					&self.active_sessions,
+					NestingSender::new(self.tx.clone(), DisputeSenderMessage::TaskFinish),
+					req,
+					&self.metrics,
+				)
+				.await?;
+				vacant.insert(send_task);
+			},
+		}
+		Ok(())
+	}
+
+	/// Receive message from a background task.
+	pub async fn on_message<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		msg: DisputeSenderMessage,
+	) -> Result<()> {
+		match msg {
+			DisputeSenderMessage::TaskFinish(msg) => {
+				let TaskFinish { candidate_hash, receiver, result } = msg;
+
+				self.metrics.on_sent_request(result.as_metrics_label());
+
+				let task = match self.disputes.get_mut(&candidate_hash) {
+					None => {
+						// Can happen when a dispute ends, with messages still in queue:
+						gum::trace!(
+							target: LOG_TARGET,
+							?result,
+							"Received `FromSendingTask::Finished` for non existing dispute."
+						);
+						return Ok(());
+					},
+					Some(task) => task,
+				};
+				task.on_finished_send(&receiver, result);
+			},
+			DisputeSenderMessage::ActiveDisputesReady(result) => {
+				let state = self.waiting_for_active_disputes.take();
+				let have_new_sessions = state.map(|s| s.have_new_sessions).unwrap_or(false);
+				let active_disputes = result?;
+				self.handle_new_active_disputes(ctx, runtime, active_disputes, have_new_sessions)
+					.await?;
+			},
+		}
+		Ok(())
+	}
+
+	/// Take care of a change in active leaves.
+	///
+	/// Update our knowledge on sessions and initiate fetching for new active disputes.
+	pub async fn update_leaves<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		update: ActiveLeavesUpdate,
+	) -> Result<()> {
+		let ActiveLeavesUpdate { activated, deactivated } = update;
+		let deactivated: HashSet<_> = deactivated.into_iter().collect();
+		self.active_heads.retain(|h| !deactivated.contains(h));
+		self.active_heads.extend(activated.into_iter().map(|l| l.hash));
+
+		let have_new_sessions = self.refresh_sessions(ctx, runtime).await?;
+
+		// Not yet waiting for data, request an update:
+		match self.waiting_for_active_disputes.take() {
+			None => {
+				self.waiting_for_active_disputes =
+					Some(WaitForActiveDisputesState { have_new_sessions });
+				let mut sender = ctx.sender().clone();
+				let mut tx = self.tx.clone();
+
+				let get_active_disputes_task = async move {
+					let result = get_active_disputes(&mut sender).await;
+					let result =
+						tx.send_message(DisputeSenderMessage::ActiveDisputesReady(result)).await;
+					if let Err(err) = result {
+						gum::debug!(
+							target: LOG_TARGET,
+							?err,
+							"Sending `DisputeSenderMessage` from background task failed."
+						);
+					}
+				};
+
+				ctx.spawn("get_active_disputes", Box::pin(get_active_disputes_task))
+					.map_err(FatalError::SpawnTask)?;
+			},
+			Some(state) => {
+				let have_new_sessions = state.have_new_sessions || have_new_sessions;
+				let new_state = WaitForActiveDisputesState { have_new_sessions };
+				self.waiting_for_active_disputes = Some(new_state);
+				gum::debug!(
+					target: LOG_TARGET,
+					"Dispute coordinator slow? We are still waiting for data on next active leaves update."
+				);
+			},
+		}
+		Ok(())
+	}
+
+	/// Handle new active disputes response.
+	///
+	/// - Initiate a retry of failed sends which are still active.
+	/// - Get new authorities to send messages to.
+	/// - Get rid of obsolete tasks and disputes.
+	///
+	/// This function ensures the `SEND_RATE_LIMIT`, therefore it might block.
+	async fn handle_new_active_disputes<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_disputes: BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>,
+		have_new_sessions: bool,
+	) -> Result<()> {
+		let active_disputes: HashSet<_> =
+			active_disputes.into_iter().map(|((_, c), _)| c).collect();
+
+		// Cleanup obsolete senders (retain keeps order of remaining elements):
+		self.disputes
+			.retain(|candidate_hash, _| active_disputes.contains(candidate_hash));
+
+		// Iterates in order of insertion:
+		let mut should_rate_limit = true;
+		for (candidate_hash, dispute) in self.disputes.iter_mut() {
+			if have_new_sessions || dispute.has_failed_sends() {
+				if should_rate_limit {
+					self.rate_limit
+						.limit("while going through new sessions/failed sends", *candidate_hash)
+						.await;
+				}
+				let sends_happened = dispute
+					.refresh_sends(ctx, runtime, &self.active_sessions, &self.metrics)
+					.await?;
+				// Only rate limit if we actually sent something out _and_ it was not just because
+				// of errors on previous sends.
+				//
+				// Reasoning: It would not be acceptable to slow down the whole subsystem, just
+				// because of a few bad peers having problems. It is actually better to risk
+				// running into their rate limit in that case and accept a minor reputation change.
+				should_rate_limit = sends_happened && have_new_sessions;
+			}
+		}
+		Ok(())
+	}
+
+	/// Make active sessions correspond to currently active heads.
+	///
+	/// Returns: true if sessions changed.
+	async fn refresh_sessions<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+	) -> Result<bool> {
+		let new_sessions = get_active_session_indices(ctx, runtime, &self.active_heads).await?;
+		let new_sessions_raw: HashSet<_> = new_sessions.keys().collect();
+		let old_sessions_raw: HashSet<_> = self.active_sessions.keys().collect();
+		let updated = new_sessions_raw != old_sessions_raw;
+		// Update in any case, so we use current heads for queries:
+		self.active_sessions = new_sessions;
+		Ok(updated)
+	}
+}
+
+/// Rate limiting logic.
+///
+/// Suitable for the sending side.
+struct RateLimit {
+	limit: Delay,
+}
+
+impl RateLimit {
+	/// Create new `RateLimit` that is immediately ready.
+	fn new() -> Self {
+		// Start with an empty duration, as there has not been any previous call.
+		Self { limit: Delay::new(Duration::new(0, 0)) }
+	}
+
+	/// Initialized with actual `SEND_RATE_LIMIT` duration.
+	fn new_limit() -> Self {
+		Self { limit: Delay::new(SEND_RATE_LIMIT) }
+	}
+
+	/// Wait until ready and prepare for next call.
+	///
+	/// String given as occasion and candidate hash are logged in case the rate limit hit.
+	async fn limit(&mut self, occasion: &'static str, candidate_hash: CandidateHash) {
+		// Wait for rate limit and add some logging:
+		let mut num_wakes: u32 = 0;
+		poll_fn(|cx| {
+			let old_limit = Pin::new(&mut self.limit);
+			match old_limit.poll(cx) {
+				Poll::Pending => {
+					gum::debug!(
+						target: LOG_TARGET,
+						?occasion,
+						?candidate_hash,
+						?num_wakes,
+						"Sending rate limit hit, slowing down requests"
+					);
+					num_wakes += 1;
+					Poll::Pending
+				},
+				Poll::Ready(()) => Poll::Ready(()),
+			}
+		})
+		.await;
+		*self = Self::new_limit();
+	}
+}
+
+/// Retrieve the currently active sessions.
+///
+/// List is all indices of all active sessions together with the head that was used for the query.
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+async fn get_active_session_indices<Context>(
+	ctx: &mut Context,
+	runtime: &mut RuntimeInfo,
+	active_heads: &Vec<Hash>,
+) -> Result<HashMap<SessionIndex, Hash>> {
+	let mut indices = HashMap::new();
+	// Iterate all heads we track as active and fetch the child' session indices.
+	for head in active_heads {
+		let session_index = runtime.get_session_index_for_child(ctx.sender(), *head).await?;
+		// Cache session info
+		if let Err(err) =
+			runtime.get_session_info_by_index(ctx.sender(), *head, session_index).await
+		{
+			gum::debug!(target: LOG_TARGET, ?err, ?session_index, "Can't cache SessionInfo");
+		}
+		indices.insert(session_index, *head);
+	}
+	Ok(indices)
+}
+
+/// Retrieve Set of active disputes from the dispute coordinator.
+async fn get_active_disputes<Sender>(
+	sender: &mut Sender,
+) -> JfyiErrorResult<BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>>
+where
+	Sender: SubsystemSender<DisputeCoordinatorMessage>,
+{
+	let (tx, rx) = oneshot::channel();
+
+	sender.send_message(DisputeCoordinatorMessage::ActiveDisputes(tx)).await;
+	rx.await.map_err(|_| JfyiError::AskActiveDisputesCanceled)
+}

pezkuwi/node/network/dispute-distribution/src/sender/send_task.rs

@@ -0,0 +1,328 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{HashMap, HashSet};
+
+use futures::{Future, FutureExt};
+
+use pezkuwi_node_network_protocol::{
+	request_response::{
+		outgoing::RequestError,
+		v1::{DisputeRequest, DisputeResponse},
+		OutgoingRequest, OutgoingResult, Recipient, Requests,
+	},
+	IfDisconnected,
+};
+use pezkuwi_node_subsystem::{messages::NetworkBridgeTxMessage, overseer};
+use pezkuwi_node_subsystem_util::{metrics, nesting_sender::NestingSender, runtime::RuntimeInfo};
+use pezkuwi_primitives::{AuthorityDiscoveryId, CandidateHash, Hash, SessionIndex, ValidatorIndex};
+
+use super::error::{FatalError, Result};
+
+use crate::{
+	metrics::{FAILED, SUCCEEDED},
+	Metrics, LOG_TARGET,
+};
+
+/// Delivery status for a particular dispute.
+///
+/// Keeps track of all the validators that have to be reached for a dispute.
+///
+/// The unit of work for a `SendTask` is an authority/validator.
+pub struct SendTask<M> {
+	/// The request we are supposed to get out to all `teyrchain` validators of the dispute's
+	/// session and to all current authorities.
+	request: DisputeRequest,
+
+	/// The set of authorities we need to send our messages to. This set will change at session
+	/// boundaries. It will always be at least the `teyrchain` validators of the session where the
+	/// dispute happened and the authorities of the current sessions as determined by active heads.
+	deliveries: HashMap<AuthorityDiscoveryId, DeliveryStatus>,
+
+	/// Whether we have any tasks failed since the last refresh.
+	has_failed_sends: bool,
+
+	/// Sender to be cloned for tasks.
+	tx: NestingSender<M, TaskFinish>,
+}
+
+/// Status of a particular vote/statement delivery to a particular validator.
+enum DeliveryStatus {
+	/// Request is still in flight.
+	Pending,
+	/// Succeeded - no need to send request to this peer anymore.
+	Succeeded,
+}
+
+/// A sending task finishes with this result:
+#[derive(Debug)]
+pub struct TaskFinish {
+	/// The candidate this task was running for.
+	pub candidate_hash: CandidateHash,
+	/// The authority the request was sent to.
+	pub receiver: AuthorityDiscoveryId,
+	/// The result of the delivery attempt.
+	pub result: TaskResult,
+}
+
+#[derive(Debug)]
+pub enum TaskResult {
+	/// Task succeeded in getting the request to its peer.
+	Succeeded,
+	/// Task was not able to get the request out to its peer.
+	///
+	/// It should be retried in that case.
+	Failed(RequestError),
+}
+
+impl TaskResult {
+	pub fn as_metrics_label(&self) -> &'static str {
+		match self {
+			Self::Succeeded => SUCCEEDED,
+			Self::Failed(_) => FAILED,
+		}
+	}
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl<M: 'static + Send + Sync> SendTask<M> {
+	/// Initiates sending a dispute message to peers.
+	///
+	/// Creation of new `SendTask`s is subject to rate limiting. As each `SendTask` will trigger
+	/// sending a message to each validator, hence for employing a per-peer rate limit, we need to
+	/// limit the construction of new `SendTask`s.
+	pub async fn new<Context>(
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+		tx: NestingSender<M, TaskFinish>,
+		request: DisputeRequest,
+		metrics: &Metrics,
+	) -> Result<Self> {
+		let mut send_task =
+			Self { request, deliveries: HashMap::new(), has_failed_sends: false, tx };
+		send_task.refresh_sends(ctx, runtime, active_sessions, metrics).await?;
+		Ok(send_task)
+	}
+
+	/// Make sure we are sending to all relevant authorities.
+	///
+	/// This function is called at construction and should also be called whenever a session change
+	/// happens and on a regular basis to ensure we are retrying failed attempts.
+	///
+	/// This might resend to validators and is thus subject to any rate limiting we might want.
+	/// Calls to this function for different instances should be rate limited according to
+	/// `SEND_RATE_LIMIT`.
+	///
+	/// Returns: `True` if this call resulted in new requests.
+	pub async fn refresh_sends<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+		metrics: &Metrics,
+	) -> Result<bool> {
+		let new_authorities = self.get_relevant_validators(ctx, runtime, active_sessions).await?;
+
+		// Note this will also contain all authorities for which sending failed previously:
+		let add_authorities: Vec<_> = new_authorities
+			.iter()
+			.filter(|a| !self.deliveries.contains_key(a))
+			.map(Clone::clone)
+			.collect();
+
+		// Get rid of dead/irrelevant tasks/statuses:
+		gum::trace!(
+			target: LOG_TARGET,
+			already_running_deliveries = ?self.deliveries.len(),
+			"Cleaning up deliveries"
+		);
+		self.deliveries.retain(|k, _| new_authorities.contains(k));
+
+		// Start any new tasks that are needed:
+		gum::trace!(
+			target: LOG_TARGET,
+			new_and_failed_authorities = ?add_authorities.len(),
+			overall_authority_set_size = ?new_authorities.len(),
+			already_running_deliveries = ?self.deliveries.len(),
+			"Starting new send requests for authorities."
+		);
+		let new_statuses =
+			send_requests(ctx, self.tx.clone(), add_authorities, self.request.clone(), metrics)
+				.await?;
+
+		let was_empty = new_statuses.is_empty();
+		gum::trace!(
+			target: LOG_TARGET,
+			sent_requests = ?new_statuses.len(),
+			"Requests dispatched."
+		);
+
+		self.has_failed_sends = false;
+		self.deliveries.extend(new_statuses.into_iter());
+		Ok(!was_empty)
+	}
+
+	/// Whether any sends have failed since the last refresh.
+	pub fn has_failed_sends(&self) -> bool {
+		self.has_failed_sends
+	}
+
+	/// Handle a finished response waiting task.
+	///
+	/// Called by `DisputeSender` upon reception of the corresponding message from our spawned
+	/// `wait_response_task`.
+	pub fn on_finished_send(&mut self, authority: &AuthorityDiscoveryId, result: TaskResult) {
+		match result {
+			TaskResult::Failed(err) => {
+				gum::trace!(
+					target: LOG_TARGET,
+					?authority,
+					candidate_hash = %self.request.0.candidate_receipt.hash(),
+					%err,
+					"Error sending dispute statements to node."
+				);
+
+				self.has_failed_sends = true;
+				// Remove state, so we know what to try again:
+				self.deliveries.remove(authority);
+			},
+			TaskResult::Succeeded => {
+				let status = match self.deliveries.get_mut(&authority) {
+					None => {
+						// Can happen when a sending became irrelevant while the response was
+						// already queued.
+						gum::debug!(
+							target: LOG_TARGET,
+							candidate = ?self.request.0.candidate_receipt.hash(),
+							?authority,
+							?result,
+							"Received `FromSendingTask::Finished` for non existing task."
+						);
+						return;
+					},
+					Some(status) => status,
+				};
+				// We are done here:
+				*status = DeliveryStatus::Succeeded;
+			},
+		}
+	}
+
+	/// Determine all validators that should receive the given dispute requests.
+	///
+	/// This is all `teyrchain` validators of the session the candidate occurred and all authorities
+	/// of all currently active sessions, determined by currently active heads.
+	async fn get_relevant_validators<Context>(
+		&self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+	) -> Result<HashSet<AuthorityDiscoveryId>> {
+		let ref_head = self.request.0.candidate_receipt.descriptor.relay_parent();
+		// Retrieve all authorities which participated in the teyrchain consensus of the session
+		// in which the candidate was backed.
+		let info = runtime
+			.get_session_info_by_index(ctx.sender(), ref_head, self.request.0.session_index)
+			.await?;
+		let session_info = &info.session_info;
+		let validator_count = session_info.validators.len();
+		let mut authorities: HashSet<_> = session_info
+			.discovery_keys
+			.iter()
+			.take(validator_count)
+			.enumerate()
+			.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
+			.map(|(_, v)| v.clone())
+			.collect();
+
+		// Retrieve all authorities for the current session as indicated by the active
+		// heads we are tracking.
+		for (session_index, head) in active_sessions.iter() {
+			let info =
+				runtime.get_session_info_by_index(ctx.sender(), *head, *session_index).await?;
+			let session_info = &info.session_info;
+			let new_set = session_info
+				.discovery_keys
+				.iter()
+				.enumerate()
+				.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
+				.map(|(_, v)| v.clone());
+			authorities.extend(new_set);
+		}
+		Ok(authorities)
+	}
+}
+
+/// Start sending of the given message to all given authorities.
+///
+/// And spawn tasks for handling the response.
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+async fn send_requests<Context, M: 'static + Send + Sync>(
+	ctx: &mut Context,
+	tx: NestingSender<M, TaskFinish>,
+	receivers: Vec<AuthorityDiscoveryId>,
+	req: DisputeRequest,
+	metrics: &Metrics,
+) -> Result<HashMap<AuthorityDiscoveryId, DeliveryStatus>> {
+	let mut statuses = HashMap::with_capacity(receivers.len());
+	let mut reqs = Vec::with_capacity(receivers.len());
+
+	for receiver in receivers {
+		let (outgoing, pending_response) =
+			OutgoingRequest::new(Recipient::Authority(receiver.clone()), req.clone());
+
+		reqs.push(Requests::DisputeSendingV1(outgoing));
+
+		let fut = wait_response_task(
+			pending_response,
+			req.0.candidate_receipt.hash(),
+			receiver.clone(),
+			tx.clone(),
+			metrics.time_dispute_request(),
+		);
+
+		ctx.spawn("dispute-sender", fut.boxed()).map_err(FatalError::SpawnTask)?;
+		statuses.insert(receiver, DeliveryStatus::Pending);
+	}
+
+	let msg = NetworkBridgeTxMessage::SendRequests(reqs, IfDisconnected::ImmediateError);
+	ctx.send_message(msg).await;
+	Ok(statuses)
+}
+
+/// Future to be spawned in a task for awaiting a response.
+async fn wait_response_task<M: 'static + Send + Sync>(
+	pending_response: impl Future<Output = OutgoingResult<DisputeResponse>>,
+	candidate_hash: CandidateHash,
+	receiver: AuthorityDiscoveryId,
+	mut tx: NestingSender<M, TaskFinish>,
+	_timer: Option<metrics::prometheus::prometheus::HistogramTimer>,
+) {
+	let result = pending_response.await;
+	let msg = match result {
+		Err(err) => TaskFinish { candidate_hash, receiver, result: TaskResult::Failed(err) },
+		Ok(DisputeResponse::Confirmed) =>
+			TaskFinish { candidate_hash, receiver, result: TaskResult::Succeeded },
+	};
+	if let Err(err) = tx.send_message(msg).await {
+		gum::debug!(
+			target: LOG_TARGET,
+			%err,
+			"Failed to notify subsystem about dispute sending result."
+		);
+	}
+}

pezkuwi/node/network/dispute-distribution/src/tests/mock.rs

@@ -0,0 +1,230 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Mock data and utility functions for unit tests in this subsystem.
+
+use std::{
+	collections::{HashMap, HashSet},
+	sync::{Arc, LazyLock},
+	time::Instant,
+};
+
+use async_trait::async_trait;
+
+use pezkuwi_node_network_protocol::{authority_discovery::AuthorityDiscovery, PeerId};
+use sc_keystore::LocalKeystore;
+use sp_application_crypto::AppCrypto;
+use sp_keyring::Sr25519Keyring;
+use sp_keystore::{Keystore, KeystorePtr};
+
+use pezkuwi_node_primitives::{DisputeMessage, SignedDisputeStatement};
+use pezkuwi_primitives::{
+	AuthorityDiscoveryId, CandidateHash, CandidateReceiptV2 as CandidateReceipt, Hash,
+	SessionIndex, SessionInfo, ValidatorId, ValidatorIndex,
+};
+use pezkuwi_primitives_test_helpers::dummy_candidate_descriptor_v2;
+
+use crate::LOG_TARGET;
+
+pub const MOCK_SESSION_INDEX: SessionIndex = 1;
+pub const MOCK_NEXT_SESSION_INDEX: SessionIndex = 2;
+pub const MOCK_VALIDATORS: [Sr25519Keyring; 6] = [
+	Sr25519Keyring::Ferdie,
+	Sr25519Keyring::Alice,
+	Sr25519Keyring::Bob,
+	Sr25519Keyring::Charlie,
+	Sr25519Keyring::Dave,
+	Sr25519Keyring::Eve,
+];
+
+pub const MOCK_AUTHORITIES_NEXT_SESSION: [Sr25519Keyring; 2] =
+	[Sr25519Keyring::One, Sr25519Keyring::Two];
+
+pub const FERDIE_INDEX: ValidatorIndex = ValidatorIndex(0);
+pub const ALICE_INDEX: ValidatorIndex = ValidatorIndex(1);
+pub const BOB_INDEX: ValidatorIndex = ValidatorIndex(2);
+pub const CHARLIE_INDEX: ValidatorIndex = ValidatorIndex(3);
+
+/// Mocked `AuthorityDiscovery` service.
+pub static MOCK_AUTHORITY_DISCOVERY: LazyLock<MockAuthorityDiscovery> =
+	LazyLock::new(|| MockAuthorityDiscovery::new());
+// Creating an innocent looking `SessionInfo` is really expensive in a debug build. Around
+// 700ms on my machine, We therefore cache those keys here:
+pub static MOCK_VALIDATORS_DISCOVERY_KEYS: LazyLock<HashMap<Sr25519Keyring, AuthorityDiscoveryId>> =
+	LazyLock::new(|| {
+		MOCK_VALIDATORS
+			.iter()
+			.chain(MOCK_AUTHORITIES_NEXT_SESSION.iter())
+			.map(|v| (*v, v.public().into()))
+			.collect()
+	});
+pub static FERDIE_DISCOVERY_KEY: LazyLock<AuthorityDiscoveryId> =
+	LazyLock::new(|| MOCK_VALIDATORS_DISCOVERY_KEYS.get(&Sr25519Keyring::Ferdie).unwrap().clone());
+
+pub static MOCK_SESSION_INFO: LazyLock<SessionInfo> = LazyLock::new(|| SessionInfo {
+	validators: MOCK_VALIDATORS.iter().take(4).map(|k| k.public().into()).collect(),
+	discovery_keys: MOCK_VALIDATORS
+		.iter()
+		.map(|k| MOCK_VALIDATORS_DISCOVERY_KEYS.get(&k).unwrap().clone())
+		.collect(),
+	assignment_keys: vec![],
+	validator_groups: Default::default(),
+	n_cores: 0,
+	zeroth_delay_tranche_width: 0,
+	relay_vrf_modulo_samples: 0,
+	n_delay_tranches: 0,
+	no_show_slots: 0,
+	needed_approvals: 0,
+	active_validator_indices: vec![],
+	dispute_period: 6,
+	random_seed: [0u8; 32],
+});
+
+/// `SessionInfo` for the second session. (No more validators, but two more authorities.
+pub static MOCK_NEXT_SESSION_INFO: LazyLock<SessionInfo> = LazyLock::new(|| SessionInfo {
+	discovery_keys: MOCK_AUTHORITIES_NEXT_SESSION
+		.iter()
+		.map(|k| MOCK_VALIDATORS_DISCOVERY_KEYS.get(&k).unwrap().clone())
+		.collect(),
+	validators: Default::default(),
+	assignment_keys: vec![],
+	validator_groups: Default::default(),
+	n_cores: 0,
+	zeroth_delay_tranche_width: 0,
+	relay_vrf_modulo_samples: 0,
+	n_delay_tranches: 0,
+	no_show_slots: 0,
+	needed_approvals: 0,
+	active_validator_indices: vec![],
+	dispute_period: 6,
+	random_seed: [0u8; 32],
+});
+
+pub fn make_candidate_receipt(relay_parent: Hash) -> CandidateReceipt {
+	CandidateReceipt {
+		descriptor: dummy_candidate_descriptor_v2(relay_parent),
+		commitments_hash: Hash::random(),
+	}
+}
+
+pub fn make_explicit_signed(
+	validator: Sr25519Keyring,
+	candidate_hash: CandidateHash,
+	valid: bool,
+) -> SignedDisputeStatement {
+	let keystore: KeystorePtr = Arc::new(LocalKeystore::in_memory());
+	Keystore::sr25519_generate_new(&*keystore, ValidatorId::ID, Some(&validator.to_seed()))
+		.expect("Insert key into keystore");
+
+	SignedDisputeStatement::sign_explicit(
+		&keystore,
+		valid,
+		candidate_hash,
+		MOCK_SESSION_INDEX,
+		validator.public().into(),
+	)
+	.expect("Keystore should be fine.")
+	.expect("Signing should work.")
+}
+
+pub fn make_dispute_message(
+	candidate: CandidateReceipt,
+	valid_validator: ValidatorIndex,
+	invalid_validator: ValidatorIndex,
+) -> DisputeMessage {
+	let candidate_hash = candidate.hash();
+	let before_request = Instant::now();
+	let valid_vote =
+		make_explicit_signed(MOCK_VALIDATORS[valid_validator.0 as usize], candidate_hash, true);
+	gum::trace!(
+		"Passed time for valid vote: {:#?}",
+		Instant::now().saturating_duration_since(before_request)
+	);
+	let before_request = Instant::now();
+	let invalid_vote =
+		make_explicit_signed(MOCK_VALIDATORS[invalid_validator.0 as usize], candidate_hash, false);
+	gum::trace!(
+		"Passed time for invalid vote: {:#?}",
+		Instant::now().saturating_duration_since(before_request)
+	);
+	DisputeMessage::from_signed_statements(
+		valid_vote,
+		valid_validator,
+		invalid_vote,
+		invalid_validator,
+		candidate,
+		&MOCK_SESSION_INFO,
+	)
+	.expect("DisputeMessage construction should work.")
+}
+
+/// Dummy `AuthorityDiscovery` service.
+#[derive(Debug, Clone)]
+pub struct MockAuthorityDiscovery {
+	peer_ids: HashMap<Sr25519Keyring, PeerId>,
+}
+
+impl MockAuthorityDiscovery {
+	pub fn new() -> Self {
+		let mut peer_ids = HashMap::new();
+		peer_ids.insert(Sr25519Keyring::Alice, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Bob, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Ferdie, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Charlie, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Dave, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Eve, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::One, PeerId::random());
+		peer_ids.insert(Sr25519Keyring::Two, PeerId::random());
+
+		Self { peer_ids }
+	}
+
+	pub fn get_peer_id_by_authority(&self, authority: Sr25519Keyring) -> PeerId {
+		*self.peer_ids.get(&authority).expect("Tester only picks valid authorities")
+	}
+}
+
+#[async_trait]
+impl AuthorityDiscovery for MockAuthorityDiscovery {
+	async fn get_addresses_by_authority_id(
+		&mut self,
+		_authority: pezkuwi_primitives::AuthorityDiscoveryId,
+	) -> Option<HashSet<sc_network::Multiaddr>> {
+		panic!("Not implemented");
+	}
+
+	async fn get_authority_ids_by_peer_id(
+		&mut self,
+		peer_id: pezkuwi_node_network_protocol::PeerId,
+	) -> Option<HashSet<pezkuwi_primitives::AuthorityDiscoveryId>> {
+		for (a, p) in self.peer_ids.iter() {
+			if p == &peer_id {
+				let result =
+					HashSet::from([MOCK_VALIDATORS_DISCOVERY_KEYS.get(&a).unwrap().clone()]);
+				gum::trace!(
+					target: LOG_TARGET,
+					%peer_id,
+					?result,
+					"Returning authority ids for peer id"
+				);
+				return Some(result);
+			}
+		}
+
+		None
+	}
+}

pezkuwi/node/network/dispute-distribution/src/tests/mod.rs

@@ -0,0 +1,901 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Subsystem unit tests
+
+use std::{
+	collections::{BTreeMap, HashSet},
+	task::Poll,
+	time::{Duration, Instant},
+};
+
+use assert_matches::assert_matches;
+use codec::{Decode, Encode};
+use futures::{
+	channel::oneshot,
+	future::{poll_fn, ready},
+	pin_mut, Future,
+};
+use futures_timer::Delay;
+
+use sc_network::{config::RequestResponseConfig, ProtocolName};
+
+use pezkuwi_node_network_protocol::{
+	request_response::{v1::DisputeRequest, IncomingRequest, ReqProtocolNames},
+	PeerId,
+};
+use sp_keyring::Sr25519Keyring;
+
+use pezkuwi_node_network_protocol::{
+	request_response::{v1::DisputeResponse, Recipient, Requests},
+	IfDisconnected,
+};
+use pezkuwi_node_primitives::DisputeStatus;
+use pezkuwi_node_subsystem::{
+	messages::{
+		AllMessages, DisputeCoordinatorMessage, DisputeDistributionMessage, ImportStatementsResult,
+		NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest,
+	},
+	ActiveLeavesUpdate, FromOrchestra, OverseerSignal,
+};
+use pezkuwi_node_subsystem_test_helpers::{
+	mock::{make_ferdie_keystore, new_leaf},
+	subsystem_test_harness, TestSubsystemContextHandle,
+};
+use pezkuwi_primitives::{
+	AuthorityDiscoveryId, Block, CandidateHash, CandidateReceiptV2 as CandidateReceipt,
+	ExecutorParams, Hash, NodeFeatures, SessionIndex, SessionInfo,
+};
+
+use self::mock::{
+	make_candidate_receipt, make_dispute_message, ALICE_INDEX, FERDIE_DISCOVERY_KEY, FERDIE_INDEX,
+	MOCK_AUTHORITY_DISCOVERY, MOCK_NEXT_SESSION_INDEX, MOCK_NEXT_SESSION_INFO, MOCK_SESSION_INDEX,
+	MOCK_SESSION_INFO,
+};
+use crate::{
+	receiver::BATCH_COLLECTING_INTERVAL,
+	tests::mock::{BOB_INDEX, CHARLIE_INDEX},
+	DisputeDistributionSubsystem, Metrics, LOG_TARGET, SEND_RATE_LIMIT,
+};
+
+/// Useful mock providers.
+pub mod mock;
+
+#[test]
+fn send_dispute_sends_dispute() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>, _req_cfg| async move {
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		send_dispute(&mut handle, candidate).await;
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn send_honors_rate_limit() {
+	sp_tracing::try_init_simple();
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>, _req_cfg| async move {
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let before_request = Instant::now();
+		send_dispute(&mut handle, candidate).await;
+		// First send should not be rate limited:
+		gum::trace!("Passed time: {:#?}", Instant::now().saturating_duration_since(before_request));
+		// This test would likely be flaky on CI:
+		//assert!(Instant::now().saturating_duration_since(before_request) < SEND_RATE_LIMIT);
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		send_dispute(&mut handle, candidate).await;
+		// Second send should be rate limited:
+		gum::trace!(
+			"Passed time for send_dispute: {:#?}",
+			Instant::now().saturating_duration_since(before_request)
+		);
+		assert!(Instant::now() - before_request >= SEND_RATE_LIMIT);
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+/// Helper for sending a new dispute to dispute-distribution sender and handling resulting messages.
+async fn send_dispute(
+	handle: &mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+	candidate: CandidateReceipt,
+) {
+	let before_request = Instant::now();
+	let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+	gum::trace!(
+		"Passed time for making message: {:#?}",
+		Instant::now().saturating_duration_since(before_request)
+	);
+	let before_request = Instant::now();
+	handle
+		.send(FromOrchestra::Communication {
+			msg: DisputeDistributionMessage::SendDispute(message.clone()),
+		})
+		.await;
+	gum::trace!(
+		"Passed time for sending message: {:#?}",
+		Instant::now().saturating_duration_since(before_request)
+	);
+
+	let expected_receivers = {
+		let info = &MOCK_SESSION_INFO;
+		info.discovery_keys
+			.clone()
+			.into_iter()
+			.filter(|a| a != &Sr25519Keyring::Ferdie.public().into())
+			.collect()
+		// All validators are also authorities in the first session, so we are
+		// done here.
+	};
+	check_sent_requests(handle, expected_receivers, true).await;
+}
+
+// Things to test:
+// x Request triggers import
+// x Subsequent imports get batched
+// x Batch gets flushed.
+// x Batch gets renewed.
+// x Non authority requests get dropped.
+// x Sending rate limit is honored.
+// x Receiving rate limit is honored.
+// x Duplicate requests on batch are dropped
+
+#[test]
+fn received_non_authorities_are_dropped() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>,
+	            mut req_cfg: RequestResponseConfig| async move {
+		let req_tx = req_cfg.inbound_queue.as_mut().unwrap();
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+
+		// Non validator request should get dropped:
+		let rx_response =
+			send_network_dispute_request(req_tx, PeerId::random(), message.clone().into()).await;
+
+		assert_matches!(
+			rx_response.await,
+			Ok(resp) => {
+				let sc_network::config::OutgoingResponse {
+					result: _,
+					reputation_changes,
+					sent_feedback: _,
+				} = resp;
+				// Peer should get punished:
+				assert_eq!(reputation_changes.len(), 1);
+			}
+		);
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn received_request_triggers_import() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>,
+	            mut req_cfg: RequestResponseConfig| async move {
+		let req_tx = req_cfg.inbound_queue.as_mut().unwrap();
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+
+		nested_network_dispute_request(
+			&mut handle,
+			req_tx,
+			MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Alice),
+			message.clone().into(),
+			ImportStatementsResult::ValidImport,
+			true,
+			move |_handle, _req_tx, _message| ready(()),
+		)
+		.await;
+
+		gum::trace!(target: LOG_TARGET, "Concluding.");
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn batching_works() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>,
+	            mut req_cfg: RequestResponseConfig| async move {
+		let req_tx = req_cfg.inbound_queue.as_mut().unwrap();
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+
+		// Initial request should get forwarded immediately:
+		nested_network_dispute_request(
+			&mut handle,
+			req_tx,
+			MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Alice),
+			message.clone().into(),
+			ImportStatementsResult::ValidImport,
+			true,
+			move |_handle, _req_tx, _message| ready(()),
+		)
+		.await;
+
+		let mut rx_responses = Vec::new();
+
+		let message = make_dispute_message(candidate.clone(), BOB_INDEX, FERDIE_INDEX);
+		let peer = MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Bob);
+		rx_responses.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+
+		let message = make_dispute_message(candidate.clone(), CHARLIE_INDEX, FERDIE_INDEX);
+		let peer = MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Charlie);
+		rx_responses.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+		gum::trace!("Imported 3 votes into batch");
+
+		Delay::new(BATCH_COLLECTING_INTERVAL);
+		gum::trace!("Batch should still be alive");
+		// Batch should still be alive (2 new votes):
+		// Let's import two more votes, but fully duplicates - should not extend batch live.
+		gum::trace!("Importing duplicate votes");
+		let mut rx_responses_duplicate = Vec::new();
+		let message = make_dispute_message(candidate.clone(), BOB_INDEX, FERDIE_INDEX);
+		let peer = MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Bob);
+		rx_responses_duplicate
+			.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+
+		let message = make_dispute_message(candidate.clone(), CHARLIE_INDEX, FERDIE_INDEX);
+		let peer = MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Charlie);
+		rx_responses_duplicate
+			.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+
+		for rx_response in rx_responses_duplicate {
+			assert_matches!(
+				rx_response.await,
+				Ok(resp) => {
+					let sc_network::config::OutgoingResponse {
+						result,
+						reputation_changes,
+						sent_feedback: _,
+					} = resp;
+					gum::trace!(
+						target: LOG_TARGET,
+						?reputation_changes,
+						"Received reputation changes."
+					);
+					// We don't punish on that.
+					assert_eq!(reputation_changes.len(), 0);
+
+					assert_matches!(result, Err(()));
+				}
+			);
+		}
+
+		Delay::new(BATCH_COLLECTING_INTERVAL).await;
+		gum::trace!("Batch should be ready now (only duplicates have been added)");
+
+		let pending_confirmation = assert_matches!(
+			handle.recv().await,
+			AllMessages::DisputeCoordinator(
+				DisputeCoordinatorMessage::ImportStatements {
+					candidate_receipt: _,
+					session,
+					statements,
+					pending_confirmation: Some(pending_confirmation),
+				}
+			) => {
+				assert_eq!(session, MOCK_SESSION_INDEX);
+				assert_eq!(statements.len(), 3);
+				pending_confirmation
+			}
+		);
+		pending_confirmation.send(ImportStatementsResult::ValidImport).unwrap();
+
+		for rx_response in rx_responses {
+			assert_matches!(
+				rx_response.await,
+				Ok(resp) => {
+					let sc_network::config::OutgoingResponse {
+						result,
+						reputation_changes: _,
+						sent_feedback,
+					} = resp;
+
+					let result = result.unwrap();
+					let decoded =
+						<DisputeResponse as Decode>::decode(&mut result.as_slice()).unwrap();
+
+					assert!(decoded == DisputeResponse::Confirmed);
+					if let Some(sent_feedback) = sent_feedback {
+						sent_feedback.send(()).unwrap();
+					}
+					gum::trace!(
+						target: LOG_TARGET,
+						"Valid import happened."
+						);
+
+				}
+			);
+		}
+
+		gum::trace!(target: LOG_TARGET, "Concluding.");
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn receive_rate_limit_is_enforced() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>,
+	            mut req_cfg: RequestResponseConfig| async move {
+		let req_tx = req_cfg.inbound_queue.as_mut().unwrap();
+		let _ = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+
+		// Initial request should get forwarded immediately:
+		nested_network_dispute_request(
+			&mut handle,
+			req_tx,
+			MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Alice),
+			message.clone().into(),
+			ImportStatementsResult::ValidImport,
+			true,
+			move |_handle, _req_tx, _message| ready(()),
+		)
+		.await;
+
+		let mut rx_responses = Vec::new();
+
+		let peer = MOCK_AUTHORITY_DISCOVERY.get_peer_id_by_authority(Sr25519Keyring::Bob);
+
+		let message = make_dispute_message(candidate.clone(), BOB_INDEX, FERDIE_INDEX);
+		rx_responses.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+
+		let message = make_dispute_message(candidate.clone(), CHARLIE_INDEX, FERDIE_INDEX);
+		rx_responses.push(send_network_dispute_request(req_tx, peer, message.clone().into()).await);
+
+		gum::trace!("Import one too much:");
+
+		let message = make_dispute_message(candidate.clone(), CHARLIE_INDEX, ALICE_INDEX);
+		let rx_response_flood =
+			send_network_dispute_request(req_tx, peer, message.clone().into()).await;
+
+		assert_matches!(
+			rx_response_flood.await,
+			Ok(resp) => {
+				let sc_network::config::OutgoingResponse {
+					result,
+					reputation_changes: _,
+					sent_feedback: _,
+				} = resp;
+				// Received error because of flood.
+				assert!(!result.is_ok());
+			}
+		);
+		gum::trace!("Need to wait 2 patch intervals:");
+		Delay::new(BATCH_COLLECTING_INTERVAL).await;
+		Delay::new(BATCH_COLLECTING_INTERVAL).await;
+
+		gum::trace!("Batch should be ready now");
+
+		let pending_confirmation = assert_matches!(
+			handle.recv().await,
+			AllMessages::DisputeCoordinator(
+				DisputeCoordinatorMessage::ImportStatements {
+					candidate_receipt: _,
+					session,
+					statements,
+					pending_confirmation: Some(pending_confirmation),
+				}
+			) => {
+				assert_eq!(session, MOCK_SESSION_INDEX);
+				// Only 3 as fourth was flood:
+				assert_eq!(statements.len(), 3);
+				pending_confirmation
+			}
+		);
+		pending_confirmation.send(ImportStatementsResult::ValidImport).unwrap();
+
+		for rx_response in rx_responses {
+			assert_matches!(
+				rx_response.await,
+				Ok(resp) => {
+					let sc_network::config::OutgoingResponse {
+						result,
+						reputation_changes: _,
+						sent_feedback,
+					} = resp;
+
+					let result = result.unwrap();
+					let decoded =
+						<DisputeResponse as Decode>::decode(&mut result.as_slice()).unwrap();
+
+					assert!(decoded == DisputeResponse::Confirmed);
+					if let Some(sent_feedback) = sent_feedback {
+						sent_feedback.send(()).unwrap();
+					}
+					gum::trace!(
+						target: LOG_TARGET,
+						"Valid import happened."
+						);
+
+				}
+			);
+		}
+
+		gum::trace!(target: LOG_TARGET, "Concluding.");
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn send_dispute_gets_cleaned_up() {
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>, _| async move {
+		let old_head = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+		handle
+			.send(FromOrchestra::Communication {
+				msg: DisputeDistributionMessage::SendDispute(message.clone()),
+			})
+			.await;
+
+		let expected_receivers = {
+			let info = &MOCK_SESSION_INFO;
+			info.discovery_keys
+				.clone()
+				.into_iter()
+				.filter(|a| a != &Sr25519Keyring::Ferdie.public().into())
+				.collect()
+			// All validators are also authorities in the first session, so we are
+			// done here.
+		};
+		check_sent_requests(&mut handle, expected_receivers, false).await;
+
+		// Give tasks a chance to finish:
+		Delay::new(Duration::from_millis(20)).await;
+
+		activate_leaf(
+			&mut handle,
+			Hash::random(),
+			Some(old_head),
+			MOCK_SESSION_INDEX,
+			None,
+			// No disputes any more:
+			BTreeMap::new(),
+		)
+		.await;
+
+		// Yield, so subsystem can make progress:
+		Delay::new(Duration::from_millis(2)).await;
+
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+#[test]
+fn dispute_retries_and_works_across_session_boundaries() {
+	sp_tracing::try_init_simple();
+	let test = |mut handle: TestSubsystemContextHandle<DisputeDistributionMessage>, _| async move {
+		let old_head = handle_subsystem_startup(&mut handle, None).await;
+
+		let relay_parent = Hash::random();
+		let candidate = make_candidate_receipt(relay_parent);
+		let message = make_dispute_message(candidate.clone(), ALICE_INDEX, FERDIE_INDEX);
+		handle
+			.send(FromOrchestra::Communication {
+				msg: DisputeDistributionMessage::SendDispute(message.clone()),
+			})
+			.await;
+
+		let expected_receivers: HashSet<_> = {
+			let info = &MOCK_SESSION_INFO;
+			info.discovery_keys
+				.clone()
+				.into_iter()
+				.filter(|a| a != &Sr25519Keyring::Ferdie.public().into())
+				.collect()
+			// All validators are also authorities in the first session, so we are
+			// done here.
+		};
+		// Requests don't get confirmed - dispute is carried over to next session.
+		check_sent_requests(&mut handle, expected_receivers.clone(), false).await;
+
+		// Give tasks a chance to finish:
+		Delay::new(Duration::from_millis(20)).await;
+
+		// Trigger retry:
+		let old_head2 = Hash::random();
+		activate_leaf(
+			&mut handle,
+			old_head2,
+			Some(old_head),
+			MOCK_SESSION_INDEX,
+			None,
+			BTreeMap::from([((MOCK_SESSION_INDEX, candidate.hash()), DisputeStatus::Active)]),
+		)
+		.await;
+
+		check_sent_requests(&mut handle, expected_receivers.clone(), false).await;
+		// Give tasks a chance to finish:
+		Delay::new(Duration::from_millis(20)).await;
+
+		// Session change:
+		activate_leaf(
+			&mut handle,
+			Hash::random(),
+			Some(old_head2),
+			MOCK_NEXT_SESSION_INDEX,
+			Some(MOCK_NEXT_SESSION_INFO.clone()),
+			BTreeMap::from([((MOCK_SESSION_INDEX, candidate.hash()), DisputeStatus::Active)]),
+		)
+		.await;
+
+		let expected_receivers = {
+			let validator_count = MOCK_SESSION_INFO.validators.len();
+			let old_validators = MOCK_SESSION_INFO
+				.discovery_keys
+				.clone()
+				.into_iter()
+				.take(validator_count)
+				.filter(|a| *a != *FERDIE_DISCOVERY_KEY);
+
+			MOCK_NEXT_SESSION_INFO
+				.discovery_keys
+				.clone()
+				.into_iter()
+				.filter(|a| *a != *FERDIE_DISCOVERY_KEY)
+				.chain(old_validators)
+				.collect()
+		};
+		check_sent_requests(&mut handle, expected_receivers, true).await;
+
+		conclude(&mut handle).await;
+	};
+	test_harness(test);
+}
+
+async fn send_network_dispute_request(
+	req_tx: &mut async_channel::Sender<sc_network::config::IncomingRequest>,
+	peer: PeerId,
+	message: DisputeRequest,
+) -> oneshot::Receiver<sc_network::config::OutgoingResponse> {
+	let (pending_response, rx_response) = oneshot::channel();
+	let req =
+		sc_network::config::IncomingRequest { peer, payload: message.encode(), pending_response };
+	req_tx.send(req).await.unwrap();
+	rx_response
+}
+
+/// Send request and handle its reactions.
+///
+/// Passed in function will be called while votes are still being imported.
+async fn nested_network_dispute_request<'a, F, O>(
+	handle: &'a mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+	req_tx: &'a mut async_channel::Sender<sc_network::config::IncomingRequest>,
+	peer: PeerId,
+	message: DisputeRequest,
+	import_result: ImportStatementsResult,
+	need_session_info: bool,
+	inner: F,
+) where
+	F: FnOnce(
+			&'a mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+			&'a mut async_channel::Sender<sc_network::config::IncomingRequest>,
+			DisputeRequest,
+		) -> O
+		+ 'a,
+	O: Future<Output = ()> + 'a,
+{
+	let rx_response = send_network_dispute_request(req_tx, peer, message.clone().into()).await;
+
+	if need_session_info {
+		// Subsystem might need `SessionInfo` for determining indices:
+		match handle.recv().await {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				_,
+				RuntimeApiRequest::SessionInfo(_, tx),
+			)) => {
+				tx.send(Ok(Some(MOCK_SESSION_INFO.clone())))
+					.expect("Receiver should stay alive.");
+			},
+			unexpected => panic!("Unexpected message {:?}", unexpected),
+		}
+		match handle.recv().await {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				_,
+				RuntimeApiRequest::SessionExecutorParams(_, tx),
+			)) => {
+				tx.send(Ok(Some(ExecutorParams::default())))
+					.expect("Receiver should stay alive.");
+			},
+			unexpected => panic!("Unexpected message {:?}", unexpected),
+		}
+
+		match handle.recv().await {
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				_,
+				RuntimeApiRequest::NodeFeatures(_, si_tx),
+			)) => {
+				si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap();
+			},
+			unexpected => panic!("Unexpected message {:?}", unexpected),
+		}
+	}
+
+	// Import should get initiated:
+	let pending_confirmation = assert_matches!(
+		handle.recv().await,
+		AllMessages::DisputeCoordinator(
+			DisputeCoordinatorMessage::ImportStatements {
+				candidate_receipt,
+				session,
+				statements,
+				pending_confirmation: Some(pending_confirmation),
+			}
+		) => {
+			let candidate_hash = candidate_receipt.hash();
+			assert_eq!(session, MOCK_SESSION_INDEX);
+			assert_eq!(candidate_hash, message.0.candidate_receipt.hash());
+			assert_eq!(statements.len(), 2);
+			pending_confirmation
+		}
+	);
+
+	// Do the inner thing:
+	inner(handle, req_tx, message).await;
+
+	// Confirm import
+	pending_confirmation.send(import_result).unwrap();
+
+	assert_matches!(
+		rx_response.await,
+		Ok(resp) => {
+			let sc_network::config::OutgoingResponse {
+				result,
+				reputation_changes,
+				sent_feedback,
+			} = resp;
+
+			match import_result {
+				ImportStatementsResult::ValidImport => {
+					let result = result.unwrap();
+					let decoded =
+						<DisputeResponse as Decode>::decode(&mut result.as_slice()).unwrap();
+
+					assert!(decoded == DisputeResponse::Confirmed);
+					if let Some(sent_feedback) = sent_feedback {
+						sent_feedback.send(()).unwrap();
+					}
+					gum::trace!(
+						target: LOG_TARGET,
+						"Valid import happened."
+					);
+
+				}
+				ImportStatementsResult::InvalidImport => {
+					// Peer should get punished:
+					assert_eq!(reputation_changes.len(), 1);
+				}
+			}
+		}
+	);
+}
+
+async fn conclude(handle: &mut TestSubsystemContextHandle<DisputeDistributionMessage>) {
+	// No more messages should be in the queue:
+	poll_fn(|ctx| {
+		let fut = handle.recv();
+		pin_mut!(fut);
+		// No requests should be initiated, as there is no longer any dispute active:
+		assert_matches!(fut.poll(ctx), Poll::Pending, "No requests expected");
+		Poll::Ready(())
+	})
+	.await;
+
+	handle.send(FromOrchestra::Signal(OverseerSignal::Conclude)).await;
+}
+
+/// Pass a `new_session` if you expect the subsystem to retrieve `SessionInfo` when given the
+/// `session_index`.
+async fn activate_leaf(
+	handle: &mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+	activate: Hash,
+	deactivate: Option<Hash>,
+	session_index: SessionIndex,
+	// New session if we expect the subsystem to request it.
+	new_session: Option<SessionInfo>,
+	// Currently active disputes to send to the subsystem.
+	active_disputes: BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>,
+) {
+	handle
+		.send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate {
+			activated: Some(new_leaf(activate, 10)),
+			deactivated: deactivate.into_iter().collect(),
+		})))
+		.await;
+	assert_matches!(
+		handle.recv().await,
+		AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+			h,
+			RuntimeApiRequest::SessionIndexForChild(tx)
+		)) => {
+			assert_eq!(h, activate);
+			tx.send(Ok(session_index)).expect("Receiver should stay alive.");
+		}
+	);
+
+	if let Some(session_info) = new_session {
+		assert_matches!(
+			handle.recv().await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				h,
+				RuntimeApiRequest::SessionInfo(session_idx, tx)
+			)) => {
+				assert_eq!(h, activate);
+				assert_eq!(session_index, session_idx);
+				tx.send(Ok(Some(session_info))).expect("Receiver should stay alive.");
+			}
+		);
+		assert_matches!(
+			handle.recv().await,
+			AllMessages::RuntimeApi(RuntimeApiMessage::Request(
+				h,
+				RuntimeApiRequest::SessionExecutorParams(session_idx, tx)
+			)) => {
+				assert_eq!(h, activate);
+				assert_eq!(session_index, session_idx);
+				tx.send(Ok(Some(ExecutorParams::default()))).expect("Receiver should stay alive.");
+			}
+		);
+		assert_matches!(
+			handle.recv().await,
+			AllMessages::RuntimeApi(
+				RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), )
+			) => {
+				si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap();
+			}
+		);
+	}
+
+	assert_matches!(
+		handle.recv().await,
+		AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::ActiveDisputes(tx)) => {
+			tx.send(active_disputes).expect("Receiver should stay alive.");
+		}
+	);
+}
+
+/// Check whether sent network bridge requests match the expectation.
+async fn check_sent_requests(
+	handle: &mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+	expected_receivers: HashSet<AuthorityDiscoveryId>,
+	confirm_receive: bool,
+) {
+	let expected_receivers: HashSet<_> =
+		expected_receivers.into_iter().map(Recipient::Authority).collect();
+
+	// Sends to concerned validators:
+	assert_matches!(
+		handle.recv().await,
+		AllMessages::NetworkBridgeTx(
+			NetworkBridgeTxMessage::SendRequests(reqs, IfDisconnected::ImmediateError)
+		) => {
+			let reqs: Vec<_> = reqs.into_iter().map(|r|
+				assert_matches!(
+					r,
+					Requests::DisputeSendingV1(req) => {req}
+				)
+			)
+			.collect();
+
+			let receivers_raw: Vec<_> = reqs.iter().map(|r| r.peer.clone()).collect();
+			let receivers: HashSet<_> = receivers_raw.clone().clone().into_iter().collect();
+			assert_eq!(receivers_raw.len(), receivers.len(), "No duplicates are expected.");
+			assert_eq!(receivers.len(), expected_receivers.len());
+			assert_eq!(receivers, expected_receivers);
+			if confirm_receive {
+				for req in reqs {
+					req.pending_response.send(
+						Ok((DisputeResponse::Confirmed.encode(), ProtocolName::from("")))
+					)
+					.expect("Subsystem should be listening for a response.");
+				}
+			}
+		}
+	);
+}
+
+/// Initialize subsystem and return request sender needed for sending incoming requests to the
+/// subsystem.
+async fn handle_subsystem_startup(
+	handle: &mut TestSubsystemContextHandle<DisputeDistributionMessage>,
+	ongoing_dispute: Option<CandidateHash>,
+) -> Hash {
+	let relay_parent = Hash::random();
+	activate_leaf(
+		handle,
+		relay_parent,
+		None,
+		MOCK_SESSION_INDEX,
+		Some(MOCK_SESSION_INFO.clone()),
+		ongoing_dispute
+			.into_iter()
+			.map(|c| ((MOCK_SESSION_INDEX, c), DisputeStatus::Active))
+			.collect(),
+	)
+	.await;
+	relay_parent
+}
+
+/// Launch subsystem and provided test function
+///
+/// which simulates the overseer.
+fn test_harness<TestFn, Fut>(test: TestFn)
+where
+	TestFn: FnOnce(
+		TestSubsystemContextHandle<DisputeDistributionMessage>,
+		RequestResponseConfig,
+	) -> Fut,
+	Fut: Future<Output = ()>,
+{
+	sp_tracing::try_init_simple();
+	let keystore = make_ferdie_keystore();
+
+	let genesis_hash = Hash::repeat_byte(0xff);
+	let req_protocol_names = ReqProtocolNames::new(&genesis_hash, None);
+	let (req_receiver, req_cfg) = IncomingRequest::get_config_receiver::<
+		Block,
+		sc_network::NetworkWorker<Block, Hash>,
+	>(&req_protocol_names);
+	let subsystem = DisputeDistributionSubsystem::new(
+		keystore,
+		req_receiver,
+		MOCK_AUTHORITY_DISCOVERY.clone(),
+		Metrics::new_dummy(),
+	);
+
+	let subsystem = |ctx| async {
+		match subsystem.run(ctx).await {
+			Ok(()) => {},
+			Err(fatal) => {
+				gum::debug!(
+					target: LOG_TARGET,
+					?fatal,
+					"Dispute distribution exited with fatal error."
+				);
+			},
+		}
+	};
+	subsystem_test_harness(|handle| test(handle, req_cfg), subsystem);
+}