Dispute Coordinator Subsystem (#3150)

* skeleton for dispute-coordinator

* add coordinator and participation message types

* begin dispute-coordinator DB

* functions for loading

* implement strongly-typed DB transaction

* add some tests for DB transaction

* core logic for pruning

* guide: update candidate-votes key for coordinator

* update candidate-votes key

* use big-endian encoding for session, and implement upper bound generator

* finish implementing pruning

* add a test for note_current_session

* define state of the subsystem itself

* barebones subsystem definition

* control flow

* more control flow

* implement session-updating logic

* trace

* control flow for message handling

* Update node/core/dispute-coordinator/src/lib.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* Update node/subsystem/src/messages.rs

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* some more control flow

* guide: remove overlay

* more control flow

* implement some DB getters

* make progress on importing statements

* add SignedDisputeStatement struct

* move ApprovalVote to shared primitives

* add a signing-payload API to explicit dispute statements

* add signing-payload to CompactStatement

* add relay-parent hash to seconded/valid dispute variatns

* correct import

* type-safe wrapper around dispute statements

* use checked dispute statement in message type

* extract rolling session window cache to subsystem-util

* extract session window tests

* approval-voting: use rolling session info cache

* reduce dispute window to match runtime in practice

* add byzantine_threshold and supermajority_threshold utilities to primitives

* integrate rolling session window

* Add PartialOrd to CandidateHash

* add Ord to CandidateHash

* implement active dispute update

* add dispute messages to AllMessages

* add dispute stubs to overseer

* inform dispute participation to participate

* implement issue_local_statement

* implement `determine_undisputed_chain`

* fix warnings

* test harness for dispute coordinator tests

* add more helpers to test harness

* add some more helpers

* some tests for dispute coordinator

* ignore wrong validator indices

* test finality voting rule constraint

* add more tests

* add variants to network bridge

* fix test compilation

* remove most dispute coordinator functionality

as of #3222 we can do most of the work within the approval voting subsystem

* Revert "remove most dispute coordinator functionality"

This reverts commit 9cd615e8eb6ca0b382cbaff525d813e753d6004e.

* Use thiserror

Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

* Update node/core/dispute-coordinator/src/lib.rs

Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

* extract tests to separate module

* address nit

* adjust run_iteration API

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: Bernhard Schuster <bernhard@ahoi.io>

polkadot/node/primitives/src/lib.rs

@@ -22,18 +22,28 @@
 
 #![deny(missing_docs)]
 
+use std::convert::TryInto;
 use std::pin::Pin;
 
 use serde::{Serialize, Deserialize};
 use futures::Future;
 use parity_scale_codec::{Decode, Encode};
+use sp_keystore::{CryptoStore, SyncCryptoStorePtr, Error as KeystoreError};
+use sp_application_crypto::AppKey;
 
 pub use sp_core::traits::SpawnNamed;
 pub use sp_consensus_babe::{
 	Epoch as BabeEpoch, BabeEpochConfiguration, AllowedSlots as BabeAllowedSlots,
 };
 
-use polkadot_primitives::v1::{BlakeTwo256, CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt, CompactStatement, EncodeAs, Hash, HashT, HeadData, Id as ParaId, OutboundHrmpMessage, PersistedValidationData, Signed, UncheckedSigned, UpwardMessage, ValidationCode, ValidatorIndex};
+use polkadot_primitives::v1::{
+	BlakeTwo256, CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt,
+	CompactStatement, EncodeAs, Hash, HashT, HeadData, Id as ParaId, OutboundHrmpMessage,
+	PersistedValidationData, Signed, UncheckedSigned, UpwardMessage, ValidationCode,
+	ValidatorIndex, ValidatorSignature, ValidDisputeStatementKind, InvalidDisputeStatementKind,
+	CandidateReceipt, ValidatorId, SessionIndex, DisputeStatement,
+};
+
 pub use polkadot_parachain::primitives::BlockData;
 
 pub mod approval;
@@ -273,3 +283,125 @@ pub fn maybe_compress_pov(pov: PoV) -> PoV {
 	let pov = PoV { block_data: BlockData(raw) };
 	pov
 }
+
+/// Tracked votes on candidates, for the purposes of dispute resolution.
+#[derive(Debug, Clone)]
+pub struct CandidateVotes {
+	/// The receipt of the candidate itself.
+	pub candidate_receipt: CandidateReceipt,
+	/// Votes of validity, sorted by validator index.
+	pub valid: Vec<(ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>,
+	/// Votes of invalidity, sorted by validator index.
+	pub invalid: Vec<(InvalidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>,
+}
+
+impl CandidateVotes {
+	/// Get the set of all validators who have votes in the set, ascending.
+	pub fn voted_indices(&self) -> Vec<ValidatorIndex> {
+		let mut v: Vec<_> = self.valid.iter().map(|x| x.1).chain(
+			self.invalid.iter().map(|x| x.1)
+		).collect();
+
+		v.sort();
+		v.dedup();
+
+		v
+	}
+}
+
+
+/// A checked dispute statement from an associated validator.
+#[derive(Debug, Clone)]
+pub struct SignedDisputeStatement {
+	dispute_statement: DisputeStatement,
+	candidate_hash: CandidateHash,
+	validator_public: ValidatorId,
+	validator_signature: ValidatorSignature,
+	session_index: SessionIndex,
+}
+
+impl SignedDisputeStatement {
+	/// Create a new `SignedDisputeStatement`, which is only possible by checking the signature.
+	pub fn new_checked(
+		dispute_statement: DisputeStatement,
+		candidate_hash: CandidateHash,
+		session_index: SessionIndex,
+		validator_public: ValidatorId,
+		validator_signature: ValidatorSignature,
+	) -> Result<Self, ()> {
+		dispute_statement.check_signature(
+			&validator_public,
+			candidate_hash,
+			session_index,
+			&validator_signature,
+		).map(|_| SignedDisputeStatement {
+			dispute_statement,
+			candidate_hash,
+			validator_public,
+			validator_signature,
+			session_index,
+		})
+	}
+
+	/// Sign this statement with the given keystore and key. Pass `valid = true` to
+	/// indicate validity of the candidate, and `valid = false` to indicate invalidity.
+	pub async fn sign_explicit(
+		keystore: &SyncCryptoStorePtr,
+		valid: bool,
+		candidate_hash: CandidateHash,
+		session_index: SessionIndex,
+		validator_public: ValidatorId,
+	) -> Result<Option<Self>, KeystoreError> {
+		let dispute_statement = if valid {
+			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit)
+		} else {
+			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit)
+		};
+
+		let data = dispute_statement.payload_data(candidate_hash, session_index);
+		let signature = CryptoStore::sign_with(
+			&**keystore,
+			ValidatorId::ID,
+			&validator_public.clone().into(),
+			&data,
+		).await?;
+
+		let signature = match signature {
+			Some(sig) => sig.try_into().map_err(|_| KeystoreError::KeyNotSupported(ValidatorId::ID))?,
+			None => return Ok(None),
+		};
+
+		Ok(Some(Self {
+			dispute_statement,
+			candidate_hash,
+			validator_public,
+			validator_signature: signature,
+			session_index,
+		}))
+	}
+
+	/// Access the underlying dispute statement
+	pub fn statement(&self) -> &DisputeStatement {
+		&self.dispute_statement
+	}
+
+	/// Access the underlying candidate hash.
+	pub fn candidate_hash(&self) -> &CandidateHash {
+		&self.candidate_hash
+	}
+
+	/// Access the underlying validator public key.
+	pub fn validator_public(&self) -> &ValidatorId {
+		&self.validator_public
+	}
+
+	/// Access the underlying validator signature.
+	pub fn validator_signature(&self) -> &ValidatorSignature {
+		&self.validator_signature
+	}
+
+	/// Access the underlying session index.
+	pub fn session_index(&self) -> SessionIndex {
+		self.session_index
+	}
+}