Approve multiple candidates with a single signature (#1191)

Initial implementation for the plan discussed here: https://github.com/paritytech/polkadot-sdk/issues/701
Built on top of https://github.com/paritytech/polkadot-sdk/pull/1178
v0: https://github.com/paritytech/polkadot/pull/7554,

## Overall idea

When approval-voting checks a candidate and is ready to advertise the
approval, defer it in a per-relay chain block until we either have
MAX_APPROVAL_COALESCE_COUNT candidates to sign or a candidate has stayed
MAX_APPROVALS_COALESCE_TICKS in the queue, in both cases we sign what
candidates we have available.

This should allow us to reduce the number of approvals messages we have
to create/send/verify. The parameters are configurable, so we should
find some values that balance:

- Security of the network: Delaying broadcasting of an approval
shouldn't but the finality at risk and to make sure that never happens
we won't delay sending a vote if we are past 2/3 from the no-show time.
- Scalability of the network: MAX_APPROVAL_COALESCE_COUNT = 1 &
MAX_APPROVALS_COALESCE_TICKS =0, is what we have now and we know from
the measurements we did on versi, it bottlenecks
approval-distribution/approval-voting when increase significantly the
number of validators and parachains
- Block storage: In case of disputes we have to import this votes on
chain and that increase the necessary storage with
MAX_APPROVAL_COALESCE_COUNT * CandidateHash per vote. Given that
disputes are not the normal way of the network functioning and we will
limit MAX_APPROVAL_COALESCE_COUNT in the single digits numbers, this
should be good enough. Alternatively, we could try to create a better
way to store this on-chain through indirection, if that's needed.

## Other fixes:
- Fixed the fact that we were sending random assignments to
non-validators, that was wrong because those won't do anything with it
and they won't gossip it either because they do not have a grid topology
set, so we would waste the random assignments.
- Added metrics to be able to debug potential no-shows and
mis-processing of approvals/assignments.

## TODO:
- [x] Get feedback, that this is moving in the right direction. @ordian
@sandreim @eskimor @burdges, let me know what you think.
- [x] More and more testing.
- [x]  Test in versi.
- [x] Make MAX_APPROVAL_COALESCE_COUNT &
MAX_APPROVAL_COALESCE_WAIT_MILLIS a parachain host configuration.
- [x] Make sure the backwards compatibility works correctly
- [x] Make sure this direction is compatible with other streams of work:
https://github.com/paritytech/polkadot-sdk/issues/635 &
https://github.com/paritytech/polkadot-sdk/issues/742
- [x] Final versi burn-in before merging

---------

Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>

polkadot/node/primitives/src/approval.rs

@@ -219,7 +219,9 @@ pub mod v2 {
 	use std::ops::BitOr;
 
 	use bitvec::{prelude::Lsb0, vec::BitVec};
-	use polkadot_primitives::{CandidateIndex, CoreIndex, Hash, ValidatorIndex};
+	use polkadot_primitives::{
+		CandidateIndex, CoreIndex, Hash, ValidatorIndex, ValidatorSignature,
+	};
 
 	/// A static context associated with producing randomness for a core.
 	pub const CORE_RANDOMNESS_CONTEXT: &[u8] = b"A&V CORE v2";
@@ -473,6 +475,59 @@ pub mod v2 {
 			})
 		}
 	}
+
+	impl From<super::v1::IndirectSignedApprovalVote> for IndirectSignedApprovalVoteV2 {
+		fn from(value: super::v1::IndirectSignedApprovalVote) -> Self {
+			Self {
+				block_hash: value.block_hash,
+				validator: value.validator,
+				candidate_indices: value.candidate_index.into(),
+				signature: value.signature,
+			}
+		}
+	}
+
+	/// Errors that can occur when trying to convert to/from approvals v1/v2
+	#[derive(Debug)]
+	pub enum ApprovalConversionError {
+		/// More than one candidate was signed.
+		MoreThanOneCandidate(usize),
+	}
+
+	impl TryFrom<IndirectSignedApprovalVoteV2> for super::v1::IndirectSignedApprovalVote {
+		type Error = ApprovalConversionError;
+
+		fn try_from(value: IndirectSignedApprovalVoteV2) -> Result<Self, Self::Error> {
+			if value.candidate_indices.count_ones() != 1 {
+				return Err(ApprovalConversionError::MoreThanOneCandidate(
+					value.candidate_indices.count_ones(),
+				))
+			}
+			Ok(Self {
+				block_hash: value.block_hash,
+				validator: value.validator,
+				candidate_index: value.candidate_indices.first_one().expect("Qed we checked above")
+					as u32,
+				signature: value.signature,
+			})
+		}
+	}
+
+	/// A signed approval vote which references the candidate indirectly via the block.
+	///
+	/// In practice, we have a look-up from block hash and candidate index to candidate hash,
+	/// so this can be transformed into a `SignedApprovalVote`.
+	#[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)]
+	pub struct IndirectSignedApprovalVoteV2 {
+		/// A block hash where the candidate appears.
+		pub block_hash: Hash,
+		/// The index of the candidate in the list of candidates fully included as-of the block.
+		pub candidate_indices: CandidateBitfield,
+		/// The validator index.
+		pub validator: ValidatorIndex,
+		/// The signature by the validator.
+		pub signature: ValidatorSignature,
+	}
 }
 
 #[cfg(test)]

polkadot/node/primitives/src/disputes/message.rs

@@ -170,7 +170,7 @@ impl DisputeMessage {
 		let valid_vote = ValidDisputeVote {
 			validator_index: valid_index,
 			signature: valid_statement.validator_signature().clone(),
-			kind: *valid_kind,
+			kind: valid_kind.clone(),
 		};
 
 		let invalid_vote = InvalidDisputeVote {

polkadot/node/primitives/src/disputes/mod.rs

@@ -46,6 +46,15 @@ pub struct SignedDisputeStatement {
 	session_index: SessionIndex,
 }
 
+/// Errors encountered while signing a dispute statement
+#[derive(Debug)]
+pub enum SignedDisputeStatementError {
+	/// Encountered a keystore error while signing
+	KeyStoreError(KeystoreError),
+	/// Could not generate signing payload
+	PayloadError,
+}
+
 /// Tracked votes on candidates, for the purposes of dispute resolution.
 #[derive(Debug, Clone)]
 pub struct CandidateVotes {
@@ -107,8 +116,9 @@ impl ValidCandidateVotes {
 				ValidDisputeStatementKind::BackingValid(_) |
 				ValidDisputeStatementKind::BackingSeconded(_) => false,
 				ValidDisputeStatementKind::Explicit |
-				ValidDisputeStatementKind::ApprovalChecking => {
-					occupied.insert((kind, sig));
+				ValidDisputeStatementKind::ApprovalChecking |
+				ValidDisputeStatementKind::ApprovalCheckingMultipleCandidates(_) => {
+					occupied.insert((kind.clone(), sig));
 					kind != occupied.get().0
 				},
 			},
@@ -213,16 +223,19 @@ impl SignedDisputeStatement {
 		candidate_hash: CandidateHash,
 		session_index: SessionIndex,
 		validator_public: ValidatorId,
-	) -> Result<Option<Self>, KeystoreError> {
+	) -> Result<Option<Self>, SignedDisputeStatementError> {
 		let dispute_statement = if valid {
 			DisputeStatement::Valid(ValidDisputeStatementKind::Explicit)
 		} else {
 			DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit)
 		};
 
-		let data = dispute_statement.payload_data(candidate_hash, session_index);
+		let data = dispute_statement
+			.payload_data(candidate_hash, session_index)
+			.map_err(|_| SignedDisputeStatementError::PayloadError)?;
 		let signature = keystore
-			.sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)?
+			.sr25519_sign(ValidatorId::ID, validator_public.as_ref(), &data)
+			.map_err(SignedDisputeStatementError::KeyStoreError)?
 			.map(|sig| Self {
 				dispute_statement,
 				candidate_hash,