collect included disputes from on-chain (#3924)

* dummy: impl another runtime API

* query the on chain disputes, and inform self

* make use of the refactor

* minro

* SPLIT ME

* write dispute values

* wip

* impl for all runtimes

* chore: fmt

* [] -> get

* fixup mock runtime

* fixup

* fixup discovery for overseer init

* chore: fmt

* spellcheck

* rename imported_on_chain_disputes -> on_chain_votes

* reduction

* make it mockable

* rename and refactor

* don't query on chain info if it's not needed

* yikes

* fmt

* fix test

* minimal fix for existing tests

* attempt to fetch the session info from the rolling window before falling back

* moved

* comments

* comments

* test for backing votes

* rename

* Update runtime/polkadot/src/lib.rs

* chore: spellcheck + dict

* chore: fmt

* fixup cache size

* add warning

* logging, rationale, less defense

* introduce new unchecked, that still checks in debug builds

* fix

* draft alt approach

* fix unused imports

* include the session

* Update node/core/dispute-coordinator/src/real/mod.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* provide where possible

* expand comment

* fixin

* fixup

* ValidityVote <-> ValidityAttestation <-> CompactStatement has a 1:1 representation

* mark TODO

* Update primitives/src/v1/mod.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* address review comments

* update docs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

polkadot/primitives/src/v0.rs

@@ -735,6 +735,7 @@ impl CompactStatement {
 /// An either implicit or explicit attestation to the validity of a parachain
 /// candidate.
 #[derive(Clone, Eq, PartialEq, Decode, Encode, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub enum ValidityAttestation {
 	/// Implicit validity attestation by issuing.
 	/// This corresponds to issuance of a `Candidate` statement.
@@ -747,6 +748,18 @@ pub enum ValidityAttestation {
 }
 
 impl ValidityAttestation {
+	/// Produce the underlying signed payload of the attestation, given the hash of the candidate,
+	/// which should be known in context.
+	pub fn to_compact_statement(&self, candidate_hash: CandidateHash) -> CompactStatement {
+		// Explicit and implicit map directly from
+		// `ValidityVote::Valid` and `ValidityVote::Issued`, and hence there is a
+		// `1:1` relationshow which enables the conversion.
+		match *self {
+			ValidityAttestation::Implicit(_) => CompactStatement::Seconded(candidate_hash),
+			ValidityAttestation::Explicit(_) => CompactStatement::Valid(candidate_hash),
+		}
+	}
+
 	/// Get a reference to the signature.
 	pub fn signature(&self) -> &ValidatorSignature {
 		match *self {

polkadot/primitives/src/v1/mod.rs

@@ -945,6 +945,22 @@ pub struct SessionInfo {
 	pub needed_approvals: u32,
 }
 
+/// Scraped runtime backing votes and resolved disputes.
+#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(PartialEq, Default, MallocSizeOf))]
+pub struct ScrapedOnChainVotes<H: Encode + Decode = Hash> {
+	/// The session in which the block was included.
+	pub session: SessionIndex,
+	/// Set of backing validators for each candidate, represented by its candidate
+	/// receipt.
+	pub backing_validators_per_candidate:
+		Vec<(CandidateReceipt<H>, Vec<(ValidatorIndex, ValidityAttestation)>)>,
+	/// On-chain-recorded set of disputes.
+	/// Note that the above `backing_validators` are
+	/// unrelated to the backers of the disputes candidates.
+	pub disputes: MultiDisputeStatementSet,
+}
+
 /// A vote of approval on a candidate.
 #[derive(Clone, RuntimeDebug)]
 pub struct ApprovalVote(pub CandidateHash);
@@ -960,7 +976,7 @@ impl ApprovalVote {
 
 sp_api::decl_runtime_apis! {
 	/// The API for querying the state of parachains on-chain.
-	pub trait ParachainHost<H: Decode = Hash, N: Encode + Decode = BlockNumber> {
+	pub trait ParachainHost<H: Encode + Decode = Hash, N: Encode + Decode = BlockNumber> {
 		/// Get the current validators.
 		fn validators() -> Vec<ValidatorId>;
 
@@ -1017,6 +1033,9 @@ sp_api::decl_runtime_apis! {
 
 		/// Get the validation code from its hash.
 		fn validation_code_by_hash(hash: ValidationCodeHash) -> Option<ValidationCode>;
+
+		/// Scrape dispute relevant from on-chain, backing votes and resolved disputes.
+		fn on_chain_votes() -> Option<ScrapedOnChainVotes<H>>;
 	}
 }
 
@@ -1182,6 +1201,7 @@ impl<H> From<ConsensusLog> for runtime_primitives::DigestItem<H> {
 ///
 /// Statements are either in favor of the candidate's validity or against it.
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub enum DisputeStatement {
 	/// A valid statement, of the given kind.
 	#[codec(index = 0)]
@@ -1251,6 +1271,7 @@ impl DisputeStatement {
 
 /// Different kinds of statements of validity on  a candidate.
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub enum ValidDisputeStatementKind {
 	/// An explicit statement issued as part of a dispute.
 	#[codec(index = 0)]
@@ -1268,6 +1289,7 @@ pub enum ValidDisputeStatementKind {
 
 /// Different kinds of statements of invalidity on a candidate.
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub enum InvalidDisputeStatementKind {
 	/// An explicit statement issued as part of a dispute.
 	#[codec(index = 0)]
@@ -1296,6 +1318,7 @@ impl ExplicitDisputeStatement {
 
 /// A set of statements about a specific candidate.
 #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)]
+#[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct DisputeStatementSet {
 	/// The candidate referenced by this set.
 	pub candidate_hash: CandidateHash,