feat: initialize Kurdistan SDK - independent fork of Polkadot SDK

pezkuwi/node/network/dispute-distribution/src/sender/error.rs

@@ -0,0 +1,73 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+//
+
+//! Error handling related code and Error/Result definitions.
+
+use pezkuwi_node_primitives::disputes::DisputeMessageCheckError;
+use pezkuwi_node_subsystem::SubsystemError;
+use pezkuwi_node_subsystem_util::runtime;
+
+#[allow(missing_docs)]
+#[fatality::fatality(splitable)]
+pub enum Error {
+	#[fatal]
+	#[error("Spawning subsystem task failed")]
+	SpawnTask(#[source] SubsystemError),
+
+	#[fatal(forward)]
+	#[error("Error while accessing runtime information")]
+	Runtime(#[from] runtime::Error),
+
+	/// We need available active heads for finding relevant authorities.
+	#[error("No active heads available - needed for finding relevant authorities.")]
+	NoActiveHeads,
+
+	/// This error likely indicates a bug in the coordinator.
+	#[error("Oneshot for asking dispute coordinator for active disputes got canceled.")]
+	AskActiveDisputesCanceled,
+
+	/// This error likely indicates a bug in the coordinator.
+	#[error("Oneshot for asking dispute coordinator for candidate votes got canceled.")]
+	AskCandidateVotesCanceled,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// We were not able to successfully construct a `DisputeMessage` from disputes votes.
+	#[error("Invalid dispute encountered")]
+	InvalidDisputeFromCoordinator(#[source] DisputeMessageCheckError),
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// We did not receive votes on both sides for `CandidateVotes` received from the coordinator.
+	#[error("Missing votes for valid dispute")]
+	MissingVotesFromCoordinator,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// `SignedDisputeStatement` could not be reconstructed from recorded statements.
+	#[error("Invalid statements from coordinator")]
+	InvalidStatementFromCoordinator,
+
+	/// This error does indicate a bug in the coordinator.
+	///
+	/// A statement's `ValidatorIndex` could not be looked up.
+	#[error("ValidatorIndex of statement could not be found")]
+	InvalidValidatorIndexFromCoordinator,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type JfyiErrorResult<T> = std::result::Result<T, JfyiError>;

pezkuwi/node/network/dispute-distribution/src/sender/mod.rs

@@ -0,0 +1,392 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{BTreeMap, HashMap, HashSet},
+	pin::Pin,
+	task::Poll,
+	time::Duration,
+};
+
+use futures::{channel::oneshot, future::poll_fn, Future};
+
+use futures_timer::Delay;
+use indexmap::{map::Entry, IndexMap};
+use pezkuwi_node_network_protocol::request_response::v1::DisputeRequest;
+use pezkuwi_node_primitives::{DisputeMessage, DisputeStatus};
+use pezkuwi_node_subsystem::{
+	messages::DisputeCoordinatorMessage, overseer, ActiveLeavesUpdate, SubsystemSender,
+};
+use pezkuwi_node_subsystem_util::{nesting_sender::NestingSender, runtime::RuntimeInfo};
+use pezkuwi_primitives::{CandidateHash, Hash, SessionIndex};
+
+/// For each ongoing dispute we have a `SendTask` which takes care of it.
+///
+/// It is going to spawn real tasks as it sees fit for getting the votes of the particular dispute
+/// out.
+///
+/// As we assume disputes have a priority, we start sending for disputes in the order
+/// `start_sender` got called.
+mod send_task;
+use send_task::SendTask;
+pub use send_task::TaskFinish;
+
+/// Error and [`Result`] type for sender.
+mod error;
+pub use error::{Error, FatalError, JfyiError, Result};
+
+use self::error::JfyiErrorResult;
+use crate::{Metrics, LOG_TARGET, SEND_RATE_LIMIT};
+
+/// Messages as sent by background tasks.
+#[derive(Debug)]
+pub enum DisputeSenderMessage {
+	/// A task finished.
+	TaskFinish(TaskFinish),
+	/// A request for active disputes to the dispute-coordinator finished.
+	ActiveDisputesReady(JfyiErrorResult<BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>>),
+}
+
+/// The `DisputeSender` keeps track of all ongoing disputes we need to send statements out.
+///
+/// For each dispute a `SendTask` is responsible for sending to the concerned validators for that
+/// particular dispute. The `DisputeSender` keeps track of those tasks, informs them about new
+/// sessions/validator sets and cleans them up when they become obsolete.
+///
+/// The unit of work for the  `DisputeSender` is a dispute, represented by `SendTask`s.
+pub struct DisputeSender<M> {
+	/// All heads we currently consider active.
+	active_heads: Vec<Hash>,
+
+	/// List of currently active sessions.
+	///
+	/// Value is the hash that was used for the query.
+	active_sessions: HashMap<SessionIndex, Hash>,
+
+	/// All ongoing dispute sending this subsystem is aware of.
+	///
+	/// Using an `IndexMap` so items can be iterated in the order of insertion.
+	disputes: IndexMap<CandidateHash, SendTask<M>>,
+
+	/// Sender to be cloned for `SendTask`s.
+	tx: NestingSender<M, DisputeSenderMessage>,
+
+	/// `Some` if we are waiting for a response `DisputeCoordinatorMessage::ActiveDisputes`.
+	waiting_for_active_disputes: Option<WaitForActiveDisputesState>,
+
+	/// Future for delaying too frequent creation of dispute sending tasks.
+	rate_limit: RateLimit,
+
+	/// Metrics for reporting stats about sent requests.
+	metrics: Metrics,
+}
+
+/// State we keep while waiting for active disputes.
+///
+/// When we send `DisputeCoordinatorMessage::ActiveDisputes`, this is the state we keep while
+/// waiting for the response.
+struct WaitForActiveDisputesState {
+	/// Have we seen any new sessions since last refresh?
+	have_new_sessions: bool,
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl<M: 'static + Send + Sync> DisputeSender<M> {
+	/// Create a new `DisputeSender` which can be used to start dispute sending.
+	pub fn new(tx: NestingSender<M, DisputeSenderMessage>, metrics: Metrics) -> Self {
+		Self {
+			active_heads: Vec::new(),
+			active_sessions: HashMap::new(),
+			disputes: IndexMap::new(),
+			tx,
+			waiting_for_active_disputes: None,
+			rate_limit: RateLimit::new(),
+			metrics,
+		}
+	}
+
+	/// Create a `SendTask` for a particular new dispute.
+	///
+	/// This function is rate-limited by `SEND_RATE_LIMIT`. It will block if called too frequently
+	/// in order to maintain the limit.
+	pub async fn start_sender<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		msg: DisputeMessage,
+	) -> Result<()> {
+		let req: DisputeRequest = msg.into();
+		let candidate_hash = req.0.candidate_receipt.hash();
+		match self.disputes.entry(candidate_hash) {
+			Entry::Occupied(_) => {
+				gum::trace!(target: LOG_TARGET, ?candidate_hash, "Dispute sending already active.");
+				return Ok(());
+			},
+			Entry::Vacant(vacant) => {
+				self.rate_limit.limit("in start_sender", candidate_hash).await;
+
+				let send_task = SendTask::new(
+					ctx,
+					runtime,
+					&self.active_sessions,
+					NestingSender::new(self.tx.clone(), DisputeSenderMessage::TaskFinish),
+					req,
+					&self.metrics,
+				)
+				.await?;
+				vacant.insert(send_task);
+			},
+		}
+		Ok(())
+	}
+
+	/// Receive message from a background task.
+	pub async fn on_message<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		msg: DisputeSenderMessage,
+	) -> Result<()> {
+		match msg {
+			DisputeSenderMessage::TaskFinish(msg) => {
+				let TaskFinish { candidate_hash, receiver, result } = msg;
+
+				self.metrics.on_sent_request(result.as_metrics_label());
+
+				let task = match self.disputes.get_mut(&candidate_hash) {
+					None => {
+						// Can happen when a dispute ends, with messages still in queue:
+						gum::trace!(
+							target: LOG_TARGET,
+							?result,
+							"Received `FromSendingTask::Finished` for non existing dispute."
+						);
+						return Ok(());
+					},
+					Some(task) => task,
+				};
+				task.on_finished_send(&receiver, result);
+			},
+			DisputeSenderMessage::ActiveDisputesReady(result) => {
+				let state = self.waiting_for_active_disputes.take();
+				let have_new_sessions = state.map(|s| s.have_new_sessions).unwrap_or(false);
+				let active_disputes = result?;
+				self.handle_new_active_disputes(ctx, runtime, active_disputes, have_new_sessions)
+					.await?;
+			},
+		}
+		Ok(())
+	}
+
+	/// Take care of a change in active leaves.
+	///
+	/// Update our knowledge on sessions and initiate fetching for new active disputes.
+	pub async fn update_leaves<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		update: ActiveLeavesUpdate,
+	) -> Result<()> {
+		let ActiveLeavesUpdate { activated, deactivated } = update;
+		let deactivated: HashSet<_> = deactivated.into_iter().collect();
+		self.active_heads.retain(|h| !deactivated.contains(h));
+		self.active_heads.extend(activated.into_iter().map(|l| l.hash));
+
+		let have_new_sessions = self.refresh_sessions(ctx, runtime).await?;
+
+		// Not yet waiting for data, request an update:
+		match self.waiting_for_active_disputes.take() {
+			None => {
+				self.waiting_for_active_disputes =
+					Some(WaitForActiveDisputesState { have_new_sessions });
+				let mut sender = ctx.sender().clone();
+				let mut tx = self.tx.clone();
+
+				let get_active_disputes_task = async move {
+					let result = get_active_disputes(&mut sender).await;
+					let result =
+						tx.send_message(DisputeSenderMessage::ActiveDisputesReady(result)).await;
+					if let Err(err) = result {
+						gum::debug!(
+							target: LOG_TARGET,
+							?err,
+							"Sending `DisputeSenderMessage` from background task failed."
+						);
+					}
+				};
+
+				ctx.spawn("get_active_disputes", Box::pin(get_active_disputes_task))
+					.map_err(FatalError::SpawnTask)?;
+			},
+			Some(state) => {
+				let have_new_sessions = state.have_new_sessions || have_new_sessions;
+				let new_state = WaitForActiveDisputesState { have_new_sessions };
+				self.waiting_for_active_disputes = Some(new_state);
+				gum::debug!(
+					target: LOG_TARGET,
+					"Dispute coordinator slow? We are still waiting for data on next active leaves update."
+				);
+			},
+		}
+		Ok(())
+	}
+
+	/// Handle new active disputes response.
+	///
+	/// - Initiate a retry of failed sends which are still active.
+	/// - Get new authorities to send messages to.
+	/// - Get rid of obsolete tasks and disputes.
+	///
+	/// This function ensures the `SEND_RATE_LIMIT`, therefore it might block.
+	async fn handle_new_active_disputes<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_disputes: BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>,
+		have_new_sessions: bool,
+	) -> Result<()> {
+		let active_disputes: HashSet<_> =
+			active_disputes.into_iter().map(|((_, c), _)| c).collect();
+
+		// Cleanup obsolete senders (retain keeps order of remaining elements):
+		self.disputes
+			.retain(|candidate_hash, _| active_disputes.contains(candidate_hash));
+
+		// Iterates in order of insertion:
+		let mut should_rate_limit = true;
+		for (candidate_hash, dispute) in self.disputes.iter_mut() {
+			if have_new_sessions || dispute.has_failed_sends() {
+				if should_rate_limit {
+					self.rate_limit
+						.limit("while going through new sessions/failed sends", *candidate_hash)
+						.await;
+				}
+				let sends_happened = dispute
+					.refresh_sends(ctx, runtime, &self.active_sessions, &self.metrics)
+					.await?;
+				// Only rate limit if we actually sent something out _and_ it was not just because
+				// of errors on previous sends.
+				//
+				// Reasoning: It would not be acceptable to slow down the whole subsystem, just
+				// because of a few bad peers having problems. It is actually better to risk
+				// running into their rate limit in that case and accept a minor reputation change.
+				should_rate_limit = sends_happened && have_new_sessions;
+			}
+		}
+		Ok(())
+	}
+
+	/// Make active sessions correspond to currently active heads.
+	///
+	/// Returns: true if sessions changed.
+	async fn refresh_sessions<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+	) -> Result<bool> {
+		let new_sessions = get_active_session_indices(ctx, runtime, &self.active_heads).await?;
+		let new_sessions_raw: HashSet<_> = new_sessions.keys().collect();
+		let old_sessions_raw: HashSet<_> = self.active_sessions.keys().collect();
+		let updated = new_sessions_raw != old_sessions_raw;
+		// Update in any case, so we use current heads for queries:
+		self.active_sessions = new_sessions;
+		Ok(updated)
+	}
+}
+
+/// Rate limiting logic.
+///
+/// Suitable for the sending side.
+struct RateLimit {
+	limit: Delay,
+}
+
+impl RateLimit {
+	/// Create new `RateLimit` that is immediately ready.
+	fn new() -> Self {
+		// Start with an empty duration, as there has not been any previous call.
+		Self { limit: Delay::new(Duration::new(0, 0)) }
+	}
+
+	/// Initialized with actual `SEND_RATE_LIMIT` duration.
+	fn new_limit() -> Self {
+		Self { limit: Delay::new(SEND_RATE_LIMIT) }
+	}
+
+	/// Wait until ready and prepare for next call.
+	///
+	/// String given as occasion and candidate hash are logged in case the rate limit hit.
+	async fn limit(&mut self, occasion: &'static str, candidate_hash: CandidateHash) {
+		// Wait for rate limit and add some logging:
+		let mut num_wakes: u32 = 0;
+		poll_fn(|cx| {
+			let old_limit = Pin::new(&mut self.limit);
+			match old_limit.poll(cx) {
+				Poll::Pending => {
+					gum::debug!(
+						target: LOG_TARGET,
+						?occasion,
+						?candidate_hash,
+						?num_wakes,
+						"Sending rate limit hit, slowing down requests"
+					);
+					num_wakes += 1;
+					Poll::Pending
+				},
+				Poll::Ready(()) => Poll::Ready(()),
+			}
+		})
+		.await;
+		*self = Self::new_limit();
+	}
+}
+
+/// Retrieve the currently active sessions.
+///
+/// List is all indices of all active sessions together with the head that was used for the query.
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+async fn get_active_session_indices<Context>(
+	ctx: &mut Context,
+	runtime: &mut RuntimeInfo,
+	active_heads: &Vec<Hash>,
+) -> Result<HashMap<SessionIndex, Hash>> {
+	let mut indices = HashMap::new();
+	// Iterate all heads we track as active and fetch the child' session indices.
+	for head in active_heads {
+		let session_index = runtime.get_session_index_for_child(ctx.sender(), *head).await?;
+		// Cache session info
+		if let Err(err) =
+			runtime.get_session_info_by_index(ctx.sender(), *head, session_index).await
+		{
+			gum::debug!(target: LOG_TARGET, ?err, ?session_index, "Can't cache SessionInfo");
+		}
+		indices.insert(session_index, *head);
+	}
+	Ok(indices)
+}
+
+/// Retrieve Set of active disputes from the dispute coordinator.
+async fn get_active_disputes<Sender>(
+	sender: &mut Sender,
+) -> JfyiErrorResult<BTreeMap<(SessionIndex, CandidateHash), DisputeStatus>>
+where
+	Sender: SubsystemSender<DisputeCoordinatorMessage>,
+{
+	let (tx, rx) = oneshot::channel();
+
+	sender.send_message(DisputeCoordinatorMessage::ActiveDisputes(tx)).await;
+	rx.await.map_err(|_| JfyiError::AskActiveDisputesCanceled)
+}

pezkuwi/node/network/dispute-distribution/src/sender/send_task.rs

@@ -0,0 +1,328 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezkuwi.
+
+// Pezkuwi is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Pezkuwi is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Pezkuwi.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::{HashMap, HashSet};
+
+use futures::{Future, FutureExt};
+
+use pezkuwi_node_network_protocol::{
+	request_response::{
+		outgoing::RequestError,
+		v1::{DisputeRequest, DisputeResponse},
+		OutgoingRequest, OutgoingResult, Recipient, Requests,
+	},
+	IfDisconnected,
+};
+use pezkuwi_node_subsystem::{messages::NetworkBridgeTxMessage, overseer};
+use pezkuwi_node_subsystem_util::{metrics, nesting_sender::NestingSender, runtime::RuntimeInfo};
+use pezkuwi_primitives::{AuthorityDiscoveryId, CandidateHash, Hash, SessionIndex, ValidatorIndex};
+
+use super::error::{FatalError, Result};
+
+use crate::{
+	metrics::{FAILED, SUCCEEDED},
+	Metrics, LOG_TARGET,
+};
+
+/// Delivery status for a particular dispute.
+///
+/// Keeps track of all the validators that have to be reached for a dispute.
+///
+/// The unit of work for a `SendTask` is an authority/validator.
+pub struct SendTask<M> {
+	/// The request we are supposed to get out to all `teyrchain` validators of the dispute's
+	/// session and to all current authorities.
+	request: DisputeRequest,
+
+	/// The set of authorities we need to send our messages to. This set will change at session
+	/// boundaries. It will always be at least the `teyrchain` validators of the session where the
+	/// dispute happened and the authorities of the current sessions as determined by active heads.
+	deliveries: HashMap<AuthorityDiscoveryId, DeliveryStatus>,
+
+	/// Whether we have any tasks failed since the last refresh.
+	has_failed_sends: bool,
+
+	/// Sender to be cloned for tasks.
+	tx: NestingSender<M, TaskFinish>,
+}
+
+/// Status of a particular vote/statement delivery to a particular validator.
+enum DeliveryStatus {
+	/// Request is still in flight.
+	Pending,
+	/// Succeeded - no need to send request to this peer anymore.
+	Succeeded,
+}
+
+/// A sending task finishes with this result:
+#[derive(Debug)]
+pub struct TaskFinish {
+	/// The candidate this task was running for.
+	pub candidate_hash: CandidateHash,
+	/// The authority the request was sent to.
+	pub receiver: AuthorityDiscoveryId,
+	/// The result of the delivery attempt.
+	pub result: TaskResult,
+}
+
+#[derive(Debug)]
+pub enum TaskResult {
+	/// Task succeeded in getting the request to its peer.
+	Succeeded,
+	/// Task was not able to get the request out to its peer.
+	///
+	/// It should be retried in that case.
+	Failed(RequestError),
+}
+
+impl TaskResult {
+	pub fn as_metrics_label(&self) -> &'static str {
+		match self {
+			Self::Succeeded => SUCCEEDED,
+			Self::Failed(_) => FAILED,
+		}
+	}
+}
+
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+impl<M: 'static + Send + Sync> SendTask<M> {
+	/// Initiates sending a dispute message to peers.
+	///
+	/// Creation of new `SendTask`s is subject to rate limiting. As each `SendTask` will trigger
+	/// sending a message to each validator, hence for employing a per-peer rate limit, we need to
+	/// limit the construction of new `SendTask`s.
+	pub async fn new<Context>(
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+		tx: NestingSender<M, TaskFinish>,
+		request: DisputeRequest,
+		metrics: &Metrics,
+	) -> Result<Self> {
+		let mut send_task =
+			Self { request, deliveries: HashMap::new(), has_failed_sends: false, tx };
+		send_task.refresh_sends(ctx, runtime, active_sessions, metrics).await?;
+		Ok(send_task)
+	}
+
+	/// Make sure we are sending to all relevant authorities.
+	///
+	/// This function is called at construction and should also be called whenever a session change
+	/// happens and on a regular basis to ensure we are retrying failed attempts.
+	///
+	/// This might resend to validators and is thus subject to any rate limiting we might want.
+	/// Calls to this function for different instances should be rate limited according to
+	/// `SEND_RATE_LIMIT`.
+	///
+	/// Returns: `True` if this call resulted in new requests.
+	pub async fn refresh_sends<Context>(
+		&mut self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+		metrics: &Metrics,
+	) -> Result<bool> {
+		let new_authorities = self.get_relevant_validators(ctx, runtime, active_sessions).await?;
+
+		// Note this will also contain all authorities for which sending failed previously:
+		let add_authorities: Vec<_> = new_authorities
+			.iter()
+			.filter(|a| !self.deliveries.contains_key(a))
+			.map(Clone::clone)
+			.collect();
+
+		// Get rid of dead/irrelevant tasks/statuses:
+		gum::trace!(
+			target: LOG_TARGET,
+			already_running_deliveries = ?self.deliveries.len(),
+			"Cleaning up deliveries"
+		);
+		self.deliveries.retain(|k, _| new_authorities.contains(k));
+
+		// Start any new tasks that are needed:
+		gum::trace!(
+			target: LOG_TARGET,
+			new_and_failed_authorities = ?add_authorities.len(),
+			overall_authority_set_size = ?new_authorities.len(),
+			already_running_deliveries = ?self.deliveries.len(),
+			"Starting new send requests for authorities."
+		);
+		let new_statuses =
+			send_requests(ctx, self.tx.clone(), add_authorities, self.request.clone(), metrics)
+				.await?;
+
+		let was_empty = new_statuses.is_empty();
+		gum::trace!(
+			target: LOG_TARGET,
+			sent_requests = ?new_statuses.len(),
+			"Requests dispatched."
+		);
+
+		self.has_failed_sends = false;
+		self.deliveries.extend(new_statuses.into_iter());
+		Ok(!was_empty)
+	}
+
+	/// Whether any sends have failed since the last refresh.
+	pub fn has_failed_sends(&self) -> bool {
+		self.has_failed_sends
+	}
+
+	/// Handle a finished response waiting task.
+	///
+	/// Called by `DisputeSender` upon reception of the corresponding message from our spawned
+	/// `wait_response_task`.
+	pub fn on_finished_send(&mut self, authority: &AuthorityDiscoveryId, result: TaskResult) {
+		match result {
+			TaskResult::Failed(err) => {
+				gum::trace!(
+					target: LOG_TARGET,
+					?authority,
+					candidate_hash = %self.request.0.candidate_receipt.hash(),
+					%err,
+					"Error sending dispute statements to node."
+				);
+
+				self.has_failed_sends = true;
+				// Remove state, so we know what to try again:
+				self.deliveries.remove(authority);
+			},
+			TaskResult::Succeeded => {
+				let status = match self.deliveries.get_mut(&authority) {
+					None => {
+						// Can happen when a sending became irrelevant while the response was
+						// already queued.
+						gum::debug!(
+							target: LOG_TARGET,
+							candidate = ?self.request.0.candidate_receipt.hash(),
+							?authority,
+							?result,
+							"Received `FromSendingTask::Finished` for non existing task."
+						);
+						return;
+					},
+					Some(status) => status,
+				};
+				// We are done here:
+				*status = DeliveryStatus::Succeeded;
+			},
+		}
+	}
+
+	/// Determine all validators that should receive the given dispute requests.
+	///
+	/// This is all `teyrchain` validators of the session the candidate occurred and all authorities
+	/// of all currently active sessions, determined by currently active heads.
+	async fn get_relevant_validators<Context>(
+		&self,
+		ctx: &mut Context,
+		runtime: &mut RuntimeInfo,
+		active_sessions: &HashMap<SessionIndex, Hash>,
+	) -> Result<HashSet<AuthorityDiscoveryId>> {
+		let ref_head = self.request.0.candidate_receipt.descriptor.relay_parent();
+		// Retrieve all authorities which participated in the teyrchain consensus of the session
+		// in which the candidate was backed.
+		let info = runtime
+			.get_session_info_by_index(ctx.sender(), ref_head, self.request.0.session_index)
+			.await?;
+		let session_info = &info.session_info;
+		let validator_count = session_info.validators.len();
+		let mut authorities: HashSet<_> = session_info
+			.discovery_keys
+			.iter()
+			.take(validator_count)
+			.enumerate()
+			.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
+			.map(|(_, v)| v.clone())
+			.collect();
+
+		// Retrieve all authorities for the current session as indicated by the active
+		// heads we are tracking.
+		for (session_index, head) in active_sessions.iter() {
+			let info =
+				runtime.get_session_info_by_index(ctx.sender(), *head, *session_index).await?;
+			let session_info = &info.session_info;
+			let new_set = session_info
+				.discovery_keys
+				.iter()
+				.enumerate()
+				.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
+				.map(|(_, v)| v.clone());
+			authorities.extend(new_set);
+		}
+		Ok(authorities)
+	}
+}
+
+/// Start sending of the given message to all given authorities.
+///
+/// And spawn tasks for handling the response.
+#[overseer::contextbounds(DisputeDistribution, prefix = self::overseer)]
+async fn send_requests<Context, M: 'static + Send + Sync>(
+	ctx: &mut Context,
+	tx: NestingSender<M, TaskFinish>,
+	receivers: Vec<AuthorityDiscoveryId>,
+	req: DisputeRequest,
+	metrics: &Metrics,
+) -> Result<HashMap<AuthorityDiscoveryId, DeliveryStatus>> {
+	let mut statuses = HashMap::with_capacity(receivers.len());
+	let mut reqs = Vec::with_capacity(receivers.len());
+
+	for receiver in receivers {
+		let (outgoing, pending_response) =
+			OutgoingRequest::new(Recipient::Authority(receiver.clone()), req.clone());
+
+		reqs.push(Requests::DisputeSendingV1(outgoing));
+
+		let fut = wait_response_task(
+			pending_response,
+			req.0.candidate_receipt.hash(),
+			receiver.clone(),
+			tx.clone(),
+			metrics.time_dispute_request(),
+		);
+
+		ctx.spawn("dispute-sender", fut.boxed()).map_err(FatalError::SpawnTask)?;
+		statuses.insert(receiver, DeliveryStatus::Pending);
+	}
+
+	let msg = NetworkBridgeTxMessage::SendRequests(reqs, IfDisconnected::ImmediateError);
+	ctx.send_message(msg).await;
+	Ok(statuses)
+}
+
+/// Future to be spawned in a task for awaiting a response.
+async fn wait_response_task<M: 'static + Send + Sync>(
+	pending_response: impl Future<Output = OutgoingResult<DisputeResponse>>,
+	candidate_hash: CandidateHash,
+	receiver: AuthorityDiscoveryId,
+	mut tx: NestingSender<M, TaskFinish>,
+	_timer: Option<metrics::prometheus::prometheus::HistogramTimer>,
+) {
+	let result = pending_response.await;
+	let msg = match result {
+		Err(err) => TaskFinish { candidate_hash, receiver, result: TaskResult::Failed(err) },
+		Ok(DisputeResponse::Confirmed) =>
+			TaskFinish { candidate_hash, receiver, result: TaskResult::Succeeded },
+	};
+	if let Err(err) = tx.send_message(msg).await {
+		gum::debug!(
+			target: LOG_TARGET,
+			%err,
+			"Failed to notify subsystem about dispute sending result."
+		);
+	}
+}