mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-07-20 04:35:41 +00:00
Dispute distribution implementation (#3282)
* Dispute protocol. * Dispute distribution protocol. * Get network requests routed. * WIP: Basic dispute sender logic. * Basic validator determination logic. * WIP: Getting things to typecheck. * Slightly larger timeout. * More typechecking stuff. * Cleanup. * Finished most of the sending logic. * Handle active leaves updates - Cleanup dead disputes - Update sends for new sessions - Retry on errors * Pass sessions in already. * Startup dispute sending. * Provide incoming decoding facilities and use them in statement-distribution. * Relaxed runtime util requirements. We only need a `SubsystemSender` not a full `SubsystemContext`. * Better usability of incoming requests. Make it possible to consume stuff without clones. * Add basic receiver functionality. * Cleanup + fixes for sender. * One more sender fix. * Start receiver. * Make sure to send responses back. * WIP: Exposed authority discovery * Make tests pass. * Fully featured receiver. * Decrease cost of `NotAValidator`. * Make `RuntimeInfo` LRU cache size configurable. * Cache more sessions. * Fix collator protocol. * Disable metrics for now. * Make dispute-distribution a proper subsystem. * Fix naming. * Code style fixes. * Factored out 4x copied mock function. * WIP: Tests. * Whitespace cleanup. * Accessor functions. * More testing. * More Debug instances. * Fix busy loop. * Working tests. * More tests. * Cleanup. * Fix build. * Basic receiving test. * Non validator message gets dropped. * More receiving tests. * Test nested and subsequent imports. * Fix spaces. * Better formatted imports. * Import cleanup. * Metrics. * Message -> MuxedMessage * Message -> MuxedMessage * More review remarks. * Add missing metrics.rs. * Fix flaky test. * Dispute coordinator - deliver confirmations. * Send out `DisputeMessage` on issue local statement. * Unwire dispute distribution. * Review remarks. * Review remarks. * Better docs.
This commit is contained in:
Robert Klotzner
@@ -0,0 +1,107 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
//
|
||||
|
||||
//! Error handling related code and Error/Result definitions.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
|
||||
use polkadot_node_subsystem_util::{Fault, runtime};
|
||||
use polkadot_subsystem::SubsystemError;
|
||||
use polkadot_node_primitives::disputes::DisputeMessageCheckError;
|
||||
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error(transparent)]
|
||||
pub struct Error(pub Fault<NonFatal, Fatal>);
|
||||
|
||||
impl From<NonFatal> for Error {
|
||||
fn from(e: NonFatal) -> Self {
|
||||
Self(Fault::from_non_fatal(e))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Fatal> for Error {
|
||||
fn from(f: Fatal) -> Self {
|
||||
Self(Fault::from_fatal(f))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<runtime::Error> for Error {
|
||||
fn from(o: runtime::Error) -> Self {
|
||||
Self(Fault::from_other(o))
|
||||
}
|
||||
}
|
||||
|
||||
/// Fatal errors of this subsystem.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Fatal {
|
||||
/// Spawning a running task failed.
|
||||
#[error("Spawning subsystem task failed")]
|
||||
SpawnTask(#[source] SubsystemError),
|
||||
|
||||
/// Errors coming from runtime::Runtime.
|
||||
#[error("Error while accessing runtime information")]
|
||||
Runtime(#[from] #[source] runtime::Fatal),
|
||||
}
|
||||
|
||||
/// Non-fatal errors of this subsystem.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum NonFatal {
|
||||
/// We need available active heads for finding relevant authorities.
|
||||
#[error("No active heads available - needed for finding relevant authorities.")]
|
||||
NoActiveHeads,
|
||||
|
||||
/// This error likely indicates a bug in the coordinator.
|
||||
#[error("Oneshot for asking dispute coordinator for active disputes got canceled.")]
|
||||
AskActiveDisputesCanceled,
|
||||
|
||||
/// This error likely indicates a bug in the coordinator.
|
||||
#[error("Oneshot for asking dispute coordinator for candidate votes got canceled.")]
|
||||
AskCandidateVotesCanceled,
|
||||
|
||||
/// This error does indicate a bug in the coordinator.
|
||||
///
|
||||
/// We were not able to successfully construct a `DisputeMessage` from disputes votes.
|
||||
#[error("Invalid dispute encountered")]
|
||||
InvalidDisputeFromCoordinator(#[source] DisputeMessageCheckError),
|
||||
|
||||
/// This error does indicate a bug in the coordinator.
|
||||
///
|
||||
/// We did not receive votes on both sides for `CandidateVotes` received from the coordinator.
|
||||
#[error("Missing votes for valid dispute")]
|
||||
MissingVotesFromCoordinator,
|
||||
|
||||
/// This error does indicate a bug in the coordinator.
|
||||
///
|
||||
/// `SignedDisputeStatement` could not be reconstructed from recorded statements.
|
||||
#[error("Invalid statements from coordinator")]
|
||||
InvalidStatementFromCoordinator,
|
||||
|
||||
/// This error does indicate a bug in the coordinator.
|
||||
///
|
||||
/// A statement's `ValidatorIndex` could not be looked up.
|
||||
#[error("ValidatorIndex of statement could not be found")]
|
||||
InvalidValidatorIndexFromCoordinator,
|
||||
|
||||
/// Errors coming from runtime::Runtime.
|
||||
#[error("Error while accessing runtime information")]
|
||||
Runtime(#[from] #[source] runtime::NonFatal),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
pub type NonFatalResult<T> = std::result::Result<T, NonFatal>;
|
||||
@@ -0,0 +1,362 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use std::collections::{HashMap, HashSet, hash_map::Entry};
|
||||
|
||||
use futures::channel::{mpsc, oneshot};
|
||||
|
||||
use polkadot_node_network_protocol::request_response::v1::DisputeRequest;
|
||||
use polkadot_node_primitives::{CandidateVotes, DisputeMessage, SignedDisputeStatement};
|
||||
use polkadot_node_subsystem_util::runtime::RuntimeInfo;
|
||||
use polkadot_primitives::v1::{CandidateHash, DisputeStatement, Hash, SessionIndex};
|
||||
use polkadot_subsystem::{
|
||||
ActiveLeavesUpdate, SubsystemContext,
|
||||
messages::{AllMessages, DisputeCoordinatorMessage}
|
||||
};
|
||||
|
||||
|
||||
/// For each ongoing dispute we have a `SendTask` which takes care of it.
|
||||
///
|
||||
/// It is going to spawn real tasks as it sees fit for getting the votes of the particular dispute
|
||||
/// out.
|
||||
mod send_task;
|
||||
use send_task::SendTask;
|
||||
pub use send_task::TaskFinish;
|
||||
|
||||
/// Error and [`Result`] type for sender
|
||||
mod error;
|
||||
pub use error::{Result, Error, Fatal, NonFatal};
|
||||
|
||||
use crate::{LOG_TARGET, Metrics};
|
||||
use self::error::NonFatalResult;
|
||||
|
||||
/// The `DisputeSender` keeps track of all ongoing disputes we need to send statements out.
|
||||
///
|
||||
/// For each dispute a `SendTask` is responsible of sending to the concerned validators for that
|
||||
/// particular dispute. The `DisputeSender` keeps track of those tasks, informs them about new
|
||||
/// sessions/validator sets and cleans them up when they become obsolete.
|
||||
pub struct DisputeSender {
|
||||
/// All heads we currently consider active.
|
||||
active_heads: Vec<Hash>,
|
||||
|
||||
/// List of currently active sessions.
|
||||
///
|
||||
/// Value is the hash that was used for the query.
|
||||
active_sessions: HashMap<SessionIndex, Hash>,
|
||||
|
||||
/// All ongoing dispute sendings this subsystem is aware of.
|
||||
disputes: HashMap<CandidateHash, SendTask>,
|
||||
|
||||
/// Sender to be cloned for `SendTask`s.
|
||||
tx: mpsc::Sender<TaskFinish>,
|
||||
|
||||
/// Metrics for reporting stats about sent requests.
|
||||
metrics: Metrics,
|
||||
}
|
||||
|
||||
impl DisputeSender
|
||||
{
|
||||
/// Create a new `DisputeSender` which can be used to start dispute sendings.
|
||||
pub fn new(tx: mpsc::Sender<TaskFinish>, metrics: Metrics) -> Self {
|
||||
Self {
|
||||
active_heads: Vec::new(),
|
||||
active_sessions: HashMap::new(),
|
||||
disputes: HashMap::new(),
|
||||
tx,
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a `SendTask` for a particular new dispute.
|
||||
pub async fn start_sender<Context: SubsystemContext>(
|
||||
&mut self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
msg: DisputeMessage,
|
||||
) -> Result<()> {
|
||||
let req: DisputeRequest = msg.into();
|
||||
let candidate_hash = req.0.candidate_receipt.hash();
|
||||
match self.disputes.entry(candidate_hash) {
|
||||
Entry::Occupied(_) => {
|
||||
tracing::trace!(
|
||||
target: LOG_TARGET,
|
||||
?candidate_hash,
|
||||
"Dispute sending already active."
|
||||
);
|
||||
return Ok(())
|
||||
}
|
||||
Entry::Vacant(vacant) => {
|
||||
let send_task = SendTask::new(
|
||||
ctx,
|
||||
runtime,
|
||||
&self.active_sessions,
|
||||
self.tx.clone(),
|
||||
req,
|
||||
)
|
||||
.await?;
|
||||
vacant.insert(send_task);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Take care of a change in active leaves.
|
||||
///
|
||||
/// - Initiate a retry of failed sends which are still active.
|
||||
/// - Get new authorities to send messages to.
|
||||
/// - Get rid of obsolete tasks and disputes.
|
||||
/// - Get dispute sending started in case we missed one for some reason (e.g. on node startup)
|
||||
pub async fn update_leaves<Context: SubsystemContext>(
|
||||
&mut self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
update: ActiveLeavesUpdate,
|
||||
) -> Result<()> {
|
||||
let ActiveLeavesUpdate { activated, deactivated } = update;
|
||||
let deactivated: HashSet<_> = deactivated.into_iter().collect();
|
||||
self.active_heads.retain(|h| !deactivated.contains(h));
|
||||
self.active_heads.extend(activated.into_iter().map(|l| l.hash));
|
||||
|
||||
let have_new_sessions = self.refresh_sessions(ctx, runtime).await?;
|
||||
|
||||
let active_disputes = get_active_disputes(ctx).await?;
|
||||
let unknown_disputes = {
|
||||
let mut disputes = active_disputes.clone();
|
||||
disputes.retain(|(_, c)| !self.disputes.contains_key(c));
|
||||
disputes
|
||||
};
|
||||
|
||||
let active_disputes: HashSet<_> = active_disputes.into_iter().map(|(_, c)| c).collect();
|
||||
|
||||
// Cleanup obsolete senders:
|
||||
self.disputes.retain(
|
||||
|candidate_hash, _| active_disputes.contains(candidate_hash)
|
||||
);
|
||||
|
||||
for dispute in self.disputes.values_mut() {
|
||||
if have_new_sessions || dispute.has_failed_sends() {
|
||||
dispute.refresh_sends(ctx, runtime, &self.active_sessions).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// This should only be non-empty on startup, but if not - we got you covered:
|
||||
for dispute in unknown_disputes {
|
||||
self.start_send_for_dispute(ctx, runtime, dispute).await?
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Receive message from a sending task.
|
||||
pub async fn on_task_message(&mut self, msg: TaskFinish) {
|
||||
|
||||
let TaskFinish { candidate_hash, receiver, result } = msg;
|
||||
|
||||
self.metrics.on_sent_request(result.as_metrics_label());
|
||||
|
||||
let task = match self.disputes.get_mut(&candidate_hash) {
|
||||
None => {
|
||||
// Can happen when a dispute ends, with messages still in queue:
|
||||
tracing::trace!(
|
||||
target: LOG_TARGET,
|
||||
?result,
|
||||
"Received `FromSendingTask::Finished` for non existing dispute."
|
||||
);
|
||||
return
|
||||
}
|
||||
Some(task) => task,
|
||||
};
|
||||
task.on_finished_send(&receiver, result);
|
||||
}
|
||||
|
||||
/// Call `start_sender` on all passed in disputes.
|
||||
///
|
||||
/// Recover necessary votes for building up `DisputeMessage` and start sending for all of them.
|
||||
async fn start_send_for_dispute<Context: SubsystemContext>(
|
||||
&mut self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
dispute: (SessionIndex, CandidateHash),
|
||||
) -> Result<()> {
|
||||
let (session_index, candidate_hash) = dispute;
|
||||
// We need some relay chain head for context for receiving session info information:
|
||||
let ref_head = self.active_sessions.values().next().ok_or(NonFatal::NoActiveHeads)?;
|
||||
let info = runtime.get_session_info_by_index(ctx.sender(), *ref_head, session_index).await?;
|
||||
let our_index = match info.validator_info.our_index {
|
||||
None => {
|
||||
tracing::trace!(
|
||||
target: LOG_TARGET,
|
||||
"Not a validator in that session - not starting dispute sending."
|
||||
);
|
||||
return Ok(())
|
||||
}
|
||||
Some(index) => index,
|
||||
};
|
||||
|
||||
let votes = match get_candidate_votes(ctx, session_index, candidate_hash).await? {
|
||||
None => {
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
?session_index,
|
||||
?candidate_hash,
|
||||
"No votes for active dispute?! - possible, due to race."
|
||||
);
|
||||
return Ok(())
|
||||
}
|
||||
Some(votes) => votes,
|
||||
};
|
||||
|
||||
let our_valid_vote = votes
|
||||
.valid
|
||||
.iter()
|
||||
.find(|(_, i, _)| *i == our_index);
|
||||
|
||||
let our_invalid_vote = votes
|
||||
.invalid
|
||||
.iter()
|
||||
.find(|(_, i, _)| *i == our_index);
|
||||
|
||||
let (valid_vote, invalid_vote) =
|
||||
if let Some(our_valid_vote) = our_valid_vote {
|
||||
// Get some invalid vote as well:
|
||||
let invalid_vote = votes
|
||||
.invalid
|
||||
.get(0)
|
||||
.ok_or(NonFatal::MissingVotesFromCoordinator)?;
|
||||
(our_valid_vote, invalid_vote)
|
||||
} else if let Some(our_invalid_vote) = our_invalid_vote {
|
||||
// Get some valid vote as well:
|
||||
let valid_vote = votes
|
||||
.valid
|
||||
.get(0)
|
||||
.ok_or(NonFatal::MissingVotesFromCoordinator)?;
|
||||
(valid_vote, our_invalid_vote)
|
||||
} else {
|
||||
return Err(From::from(NonFatal::MissingVotesFromCoordinator))
|
||||
}
|
||||
;
|
||||
let (kind, valid_index, signature) = valid_vote;
|
||||
let valid_public = info
|
||||
.session_info
|
||||
.validators
|
||||
.get(valid_index.0 as usize)
|
||||
.ok_or(NonFatal::InvalidStatementFromCoordinator)?;
|
||||
let valid_signed = SignedDisputeStatement::new_checked(
|
||||
DisputeStatement::Valid(kind.clone()),
|
||||
candidate_hash,
|
||||
session_index,
|
||||
valid_public.clone(),
|
||||
signature.clone(),
|
||||
)
|
||||
.map_err(|()| NonFatal::InvalidStatementFromCoordinator)?;
|
||||
|
||||
let (kind, invalid_index, signature) = invalid_vote;
|
||||
let invalid_public = info
|
||||
.session_info
|
||||
.validators
|
||||
.get(invalid_index.0 as usize)
|
||||
.ok_or(NonFatal::InvalidValidatorIndexFromCoordinator)?;
|
||||
let invalid_signed = SignedDisputeStatement::new_checked(
|
||||
DisputeStatement::Invalid(kind.clone()),
|
||||
candidate_hash,
|
||||
session_index,
|
||||
invalid_public.clone(),
|
||||
signature.clone(),
|
||||
)
|
||||
.map_err(|()| NonFatal::InvalidValidatorIndexFromCoordinator)?;
|
||||
|
||||
// Reconstructing the checked signed dispute statements is hardly useful here and wasteful,
|
||||
// but I don't want to enable a bypass for the below smart constructor and this code path
|
||||
// is supposed to be only hit on startup basically.
|
||||
//
|
||||
// Revisit this decision when the `from_signed_statements` is unneded for the normal code
|
||||
// path as well.
|
||||
let message = DisputeMessage::from_signed_statements(
|
||||
valid_signed,
|
||||
*valid_index,
|
||||
invalid_signed,
|
||||
*invalid_index,
|
||||
votes.candidate_receipt,
|
||||
&info.session_info
|
||||
)
|
||||
.map_err(NonFatal::InvalidDisputeFromCoordinator)?;
|
||||
|
||||
// Finally, get the party started:
|
||||
self.start_sender(ctx, runtime, message).await
|
||||
}
|
||||
|
||||
/// Make active sessions correspond to currently active heads.
|
||||
///
|
||||
/// Returns: true if sessions changed.
|
||||
async fn refresh_sessions<Context: SubsystemContext>(
|
||||
&mut self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
) -> Result<bool> {
|
||||
let new_sessions = get_active_session_indeces(ctx, runtime, &self.active_heads).await?;
|
||||
let new_sessions_raw: HashSet<_> = new_sessions.keys().collect();
|
||||
let old_sessions_raw: HashSet<_> = self.active_sessions.keys().collect();
|
||||
let updated = new_sessions_raw != old_sessions_raw;
|
||||
// Update in any case, so we use current heads for queries:
|
||||
self.active_sessions = new_sessions;
|
||||
Ok(updated)
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieve the currently active sessions.
|
||||
///
|
||||
/// List is all indeces of all active sessions together with the head that was used for the query.
|
||||
async fn get_active_session_indeces<Context: SubsystemContext>(
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
active_heads: &Vec<Hash>,
|
||||
) -> Result<HashMap<SessionIndex, Hash>> {
|
||||
let mut indeces = HashMap::new();
|
||||
for head in active_heads {
|
||||
let session_index = runtime.get_session_index(ctx.sender(), *head).await?;
|
||||
indeces.insert(session_index, *head);
|
||||
}
|
||||
Ok(indeces)
|
||||
}
|
||||
|
||||
/// Retrieve Set of active disputes from the dispute coordinator.
|
||||
async fn get_active_disputes<Context: SubsystemContext>(ctx: &mut Context)
|
||||
-> NonFatalResult<Vec<(SessionIndex, CandidateHash)>> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
ctx.send_message(AllMessages::DisputeCoordinator(
|
||||
DisputeCoordinatorMessage::ActiveDisputes(tx)
|
||||
))
|
||||
.await;
|
||||
rx.await.map_err(|_| NonFatal::AskActiveDisputesCanceled)
|
||||
}
|
||||
|
||||
/// Get all locally available dispute votes for a given dispute.
|
||||
async fn get_candidate_votes<Context: SubsystemContext>(
|
||||
ctx: &mut Context,
|
||||
session_index: SessionIndex,
|
||||
candidate_hash: CandidateHash,
|
||||
) -> NonFatalResult<Option<CandidateVotes>> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
ctx.send_message(AllMessages::DisputeCoordinator(
|
||||
DisputeCoordinatorMessage::QueryCandidateVotes(
|
||||
session_index,
|
||||
candidate_hash,
|
||||
tx
|
||||
)
|
||||
))
|
||||
.await;
|
||||
rx.await.map_err(|_| NonFatal::AskCandidateVotesCanceled)
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
// Copyright 2021 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Polkadot.
|
||||
|
||||
// Polkadot is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Polkadot is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use futures::Future;
|
||||
use futures::FutureExt;
|
||||
use futures::SinkExt;
|
||||
use futures::channel::mpsc;
|
||||
use futures::future::RemoteHandle;
|
||||
|
||||
use polkadot_node_network_protocol::{
|
||||
IfDisconnected,
|
||||
request_response::{
|
||||
OutgoingRequest, OutgoingResult, Recipient, Requests,
|
||||
v1::{DisputeRequest, DisputeResponse},
|
||||
}
|
||||
};
|
||||
use polkadot_node_subsystem_util::runtime::RuntimeInfo;
|
||||
use polkadot_primitives::v1::{
|
||||
AuthorityDiscoveryId, CandidateHash, Hash, SessionIndex, ValidatorIndex,
|
||||
};
|
||||
use polkadot_subsystem::{
|
||||
SubsystemContext,
|
||||
messages::{AllMessages, NetworkBridgeMessage},
|
||||
};
|
||||
|
||||
use super::error::{Fatal, Result};
|
||||
|
||||
use crate::LOG_TARGET;
|
||||
use crate::metrics::FAILED;
|
||||
use crate::metrics::SUCCEEDED;
|
||||
|
||||
/// Delivery status for a particular dispute.
|
||||
///
|
||||
/// Keeps track of all the validators that have to be reached for a dispute.
|
||||
pub struct SendTask {
|
||||
/// The request we are supposed to get out to all parachain validators of the dispute's session
|
||||
/// and to all current authorities.
|
||||
request: DisputeRequest,
|
||||
|
||||
/// The set of authorities we need to send our messages to. This set will change at session
|
||||
/// boundaries. It will always be at least the parachain validators of the session where the
|
||||
/// dispute happened and the authorities of the current sessions as determined by active heads.
|
||||
deliveries: HashMap<AuthorityDiscoveryId, DeliveryStatus>,
|
||||
|
||||
/// Whether or not we have any tasks failed since the last refresh.
|
||||
has_failed_sends: bool,
|
||||
|
||||
/// Sender to be cloned for tasks.
|
||||
tx: mpsc::Sender<TaskFinish>,
|
||||
}
|
||||
|
||||
/// Status of a particular vote/statement delivery to a particular validator.
|
||||
enum DeliveryStatus {
|
||||
/// Request is still in flight.
|
||||
Pending(RemoteHandle<()>),
|
||||
/// Succeeded - no need to send request to this peer anymore.
|
||||
Succeeded,
|
||||
}
|
||||
|
||||
/// A sending task finishes with this result:
|
||||
#[derive(Debug)]
|
||||
pub struct TaskFinish {
|
||||
/// The candidate this task was running for.
|
||||
pub candidate_hash: CandidateHash,
|
||||
/// The authority the request was sent to.
|
||||
pub receiver: AuthorityDiscoveryId,
|
||||
/// The result of the delivery attempt.
|
||||
pub result: TaskResult,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum TaskResult {
|
||||
/// Task succeeded in getting the request to its peer.
|
||||
Succeeded,
|
||||
/// Task was not able to get the request out to its peer.
|
||||
///
|
||||
/// It should be retried in that case.
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl TaskResult {
|
||||
pub fn as_metrics_label(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Succeeded => SUCCEEDED,
|
||||
Self::Failed => FAILED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SendTask
|
||||
{
|
||||
/// Initiates sending a dispute message to peers.
|
||||
pub async fn new<Context: SubsystemContext>(
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
active_sessions: &HashMap<SessionIndex,Hash>,
|
||||
tx: mpsc::Sender<TaskFinish>,
|
||||
request: DisputeRequest,
|
||||
) -> Result<Self> {
|
||||
let mut send_task = Self {
|
||||
request,
|
||||
deliveries: HashMap::new(),
|
||||
has_failed_sends: false,
|
||||
tx,
|
||||
};
|
||||
send_task.refresh_sends(
|
||||
ctx,
|
||||
runtime,
|
||||
active_sessions,
|
||||
).await?;
|
||||
Ok(send_task)
|
||||
}
|
||||
|
||||
/// Make sure we are sending to all relevant authorities.
|
||||
///
|
||||
/// This function is called at construction and should also be called whenever a session change
|
||||
/// happens and on a regular basis to ensure we are retrying failed attempts.
|
||||
pub async fn refresh_sends<Context: SubsystemContext>(
|
||||
&mut self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
active_sessions: &HashMap<SessionIndex, Hash>,
|
||||
) -> Result<()> {
|
||||
let new_authorities = self.get_relevant_validators(ctx, runtime, active_sessions).await?;
|
||||
|
||||
let add_authorities = new_authorities
|
||||
.iter()
|
||||
.filter(|a| !self.deliveries.contains_key(a))
|
||||
.map(Clone::clone)
|
||||
.collect();
|
||||
|
||||
// Get rid of dead/irrelevant tasks/statuses:
|
||||
self.deliveries.retain(|k, _| new_authorities.contains(k));
|
||||
|
||||
// Start any new tasks that are needed:
|
||||
let new_statuses = send_requests(
|
||||
ctx,
|
||||
self.tx.clone(),
|
||||
add_authorities,
|
||||
self.request.clone(),
|
||||
).await?;
|
||||
|
||||
self.deliveries.extend(new_statuses.into_iter());
|
||||
self.has_failed_sends = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether or not any sends have failed since the last refreshed.
|
||||
pub fn has_failed_sends(&self) -> bool {
|
||||
self.has_failed_sends
|
||||
}
|
||||
|
||||
/// Handle a finished response waiting task.
|
||||
pub fn on_finished_send(&mut self, authority: &AuthorityDiscoveryId, result: TaskResult) {
|
||||
match result {
|
||||
TaskResult::Failed => {
|
||||
tracing::warn!(
|
||||
target: LOG_TARGET,
|
||||
candidate = ?self.request.0.candidate_receipt.hash(),
|
||||
?authority,
|
||||
"Could not get our message out! If this keeps happening, then check chain whether the dispute made it there."
|
||||
);
|
||||
self.has_failed_sends = true;
|
||||
// Remove state, so we know what to try again:
|
||||
self.deliveries.remove(authority);
|
||||
}
|
||||
TaskResult::Succeeded => {
|
||||
let status = match self.deliveries.get_mut(&authority) {
|
||||
None => {
|
||||
// Can happen when a sending became irrelevant while the response was already
|
||||
// queued.
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
candidate = ?self.request.0.candidate_receipt.hash(),
|
||||
?authority,
|
||||
?result,
|
||||
"Received `FromSendingTask::Finished` for non existing task."
|
||||
);
|
||||
return
|
||||
}
|
||||
Some(status) => status,
|
||||
};
|
||||
// We are done here:
|
||||
*status = DeliveryStatus::Succeeded;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Determine all validators that should receive the given dispute requests.
|
||||
///
|
||||
/// This is all parachain validators of the session the candidate occurred and all authorities
|
||||
/// of all currently active sessions, determined by currently active heads.
|
||||
async fn get_relevant_validators<Context: SubsystemContext>(
|
||||
&self,
|
||||
ctx: &mut Context,
|
||||
runtime: &mut RuntimeInfo,
|
||||
active_sessions: &HashMap<SessionIndex, Hash>,
|
||||
) -> Result<HashSet<AuthorityDiscoveryId>> {
|
||||
let ref_head = self.request.0.candidate_receipt.descriptor.relay_parent;
|
||||
// Parachain validators:
|
||||
let info = runtime
|
||||
.get_session_info_by_index(ctx.sender(), ref_head, self.request.0.session_index)
|
||||
.await?;
|
||||
let session_info = &info.session_info;
|
||||
let validator_count = session_info.validators.len();
|
||||
let mut authorities: HashSet<_> = session_info
|
||||
.discovery_keys
|
||||
.iter()
|
||||
.take(validator_count)
|
||||
.enumerate()
|
||||
.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
|
||||
.map(|(_, v)| v.clone())
|
||||
.collect();
|
||||
|
||||
// Current authorities:
|
||||
for (session_index, head) in active_sessions.iter() {
|
||||
let info = runtime.get_session_info_by_index(ctx.sender(), *head, *session_index).await?;
|
||||
let session_info = &info.session_info;
|
||||
let new_set = session_info
|
||||
.discovery_keys
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| Some(ValidatorIndex(*i as _)) != info.validator_info.our_index)
|
||||
.map(|(_, v)| v.clone());
|
||||
authorities.extend(new_set);
|
||||
}
|
||||
Ok(authorities)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Start sending of the given msg to all given authorities.
|
||||
///
|
||||
/// And spawn tasks for handling the response.
|
||||
async fn send_requests<Context: SubsystemContext>(
|
||||
ctx: &mut Context,
|
||||
tx: mpsc::Sender<TaskFinish>,
|
||||
receivers: Vec<AuthorityDiscoveryId>,
|
||||
req: DisputeRequest,
|
||||
) -> Result<HashMap<AuthorityDiscoveryId, DeliveryStatus>> {
|
||||
let mut statuses = HashMap::with_capacity(receivers.len());
|
||||
let mut reqs = Vec::with_capacity(receivers.len());
|
||||
|
||||
for receiver in receivers {
|
||||
let (outgoing, pending_response) = OutgoingRequest::new(
|
||||
Recipient::Authority(receiver.clone()),
|
||||
req.clone(),
|
||||
);
|
||||
|
||||
reqs.push(Requests::DisputeSending(outgoing));
|
||||
|
||||
let fut = wait_response_task(
|
||||
pending_response,
|
||||
req.0.candidate_receipt.hash(),
|
||||
receiver.clone(),
|
||||
tx.clone(),
|
||||
);
|
||||
|
||||
let (remote, remote_handle) = fut.remote_handle();
|
||||
ctx.spawn("dispute-sender", remote.boxed())
|
||||
.map_err(Fatal::SpawnTask)?;
|
||||
statuses.insert(receiver, DeliveryStatus::Pending(remote_handle));
|
||||
}
|
||||
|
||||
let msg = NetworkBridgeMessage::SendRequests(
|
||||
reqs,
|
||||
// We should be connected, but the hell - if not, try!
|
||||
IfDisconnected::TryConnect,
|
||||
);
|
||||
ctx.send_message(AllMessages::NetworkBridge(msg)).await;
|
||||
Ok(statuses)
|
||||
}
|
||||
|
||||
/// Future to be spawned in a task for awaiting a response.
|
||||
async fn wait_response_task(
|
||||
pending_response: impl Future<Output = OutgoingResult<DisputeResponse>>,
|
||||
candidate_hash: CandidateHash,
|
||||
receiver: AuthorityDiscoveryId,
|
||||
mut tx: mpsc::Sender<TaskFinish>,
|
||||
) {
|
||||
let result = pending_response.await;
|
||||
let msg = match result {
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
target: LOG_TARGET,
|
||||
%candidate_hash,
|
||||
%receiver,
|
||||
%err,
|
||||
"Error sending dispute statements to node."
|
||||
);
|
||||
TaskFinish { candidate_hash, receiver, result: TaskResult::Failed}
|
||||
}
|
||||
Ok(DisputeResponse::Confirmed) => {
|
||||
tracing::trace!(
|
||||
target: LOG_TARGET,
|
||||
%candidate_hash,
|
||||
%receiver,
|
||||
"Sending dispute message succeeded"
|
||||
);
|
||||
TaskFinish { candidate_hash, receiver, result: TaskResult::Succeeded }
|
||||
}
|
||||
};
|
||||
if let Err(err) = tx.feed(msg).await {
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
%err,
|
||||
"Failed to notify susystem about dispute sending result."
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user