Approve multiple candidates with a single signature (#1191)

Initial implementation for the plan discussed here: https://github.com/paritytech/polkadot-sdk/issues/701
Built on top of https://github.com/paritytech/polkadot-sdk/pull/1178
v0: https://github.com/paritytech/polkadot/pull/7554,

## Overall idea

When approval-voting checks a candidate and is ready to advertise the
approval, defer it in a per-relay chain block until we either have
MAX_APPROVAL_COALESCE_COUNT candidates to sign or a candidate has stayed
MAX_APPROVALS_COALESCE_TICKS in the queue, in both cases we sign what
candidates we have available.

This should allow us to reduce the number of approvals messages we have
to create/send/verify. The parameters are configurable, so we should
find some values that balance:

- Security of the network: Delaying broadcasting of an approval
shouldn't but the finality at risk and to make sure that never happens
we won't delay sending a vote if we are past 2/3 from the no-show time.
- Scalability of the network: MAX_APPROVAL_COALESCE_COUNT = 1 &
MAX_APPROVALS_COALESCE_TICKS =0, is what we have now and we know from
the measurements we did on versi, it bottlenecks
approval-distribution/approval-voting when increase significantly the
number of validators and parachains
- Block storage: In case of disputes we have to import this votes on
chain and that increase the necessary storage with
MAX_APPROVAL_COALESCE_COUNT * CandidateHash per vote. Given that
disputes are not the normal way of the network functioning and we will
limit MAX_APPROVAL_COALESCE_COUNT in the single digits numbers, this
should be good enough. Alternatively, we could try to create a better
way to store this on-chain through indirection, if that's needed.

## Other fixes:
- Fixed the fact that we were sending random assignments to
non-validators, that was wrong because those won't do anything with it
and they won't gossip it either because they do not have a grid topology
set, so we would waste the random assignments.
- Added metrics to be able to debug potential no-shows and
mis-processing of approvals/assignments.

## TODO:
- [x] Get feedback, that this is moving in the right direction. @ordian
@sandreim @eskimor @burdges, let me know what you think.
- [x] More and more testing.
- [x]  Test in versi.
- [x] Make MAX_APPROVAL_COALESCE_COUNT &
MAX_APPROVAL_COALESCE_WAIT_MILLIS a parachain host configuration.
- [x] Make sure the backwards compatibility works correctly
- [x] Make sure this direction is compatible with other streams of work:
https://github.com/paritytech/polkadot-sdk/issues/635 &
https://github.com/paritytech/polkadot-sdk/issues/742
- [x] Final versi burn-in before merging

---------

Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>

polkadot/node/network/protocol/src/lib.rs

@@ -253,29 +253,29 @@ impl View {
 
 /// A protocol-versioned type.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Versioned<V1, V2, VStaging = V2> {
+pub enum Versioned<V1, V2, V3 = V2> {
 	/// V1 type.
 	V1(V1),
 	/// V2 type.
 	V2(V2),
-	/// VStaging type
-	VStaging(VStaging),
+	/// V3 type
+	V3(V3),
 }
 
-impl<V1: Clone, V2: Clone, VStaging: Clone> Versioned<&'_ V1, &'_ V2, &'_ VStaging> {
+impl<V1: Clone, V2: Clone, V3: Clone> Versioned<&'_ V1, &'_ V2, &'_ V3> {
 	/// Convert to a fully-owned version of the message.
-	pub fn clone_inner(&self) -> Versioned<V1, V2, VStaging> {
+	pub fn clone_inner(&self) -> Versioned<V1, V2, V3> {
 		match *self {
 			Versioned::V1(inner) => Versioned::V1(inner.clone()),
 			Versioned::V2(inner) => Versioned::V2(inner.clone()),
-			Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()),
+			Versioned::V3(inner) => Versioned::V3(inner.clone()),
 		}
 	}
 }
 
 /// All supported versions of the validation protocol message.
 pub type VersionedValidationProtocol =
-	Versioned<v1::ValidationProtocol, v2::ValidationProtocol, vstaging::ValidationProtocol>;
+	Versioned<v1::ValidationProtocol, v2::ValidationProtocol, v3::ValidationProtocol>;
 
 impl From<v1::ValidationProtocol> for VersionedValidationProtocol {
 	fn from(v1: v1::ValidationProtocol) -> Self {
@@ -289,9 +289,9 @@ impl From<v2::ValidationProtocol> for VersionedValidationProtocol {
 	}
 }
 
-impl From<vstaging::ValidationProtocol> for VersionedValidationProtocol {
-	fn from(vstaging: vstaging::ValidationProtocol) -> Self {
-		VersionedValidationProtocol::VStaging(vstaging)
+impl From<v3::ValidationProtocol> for VersionedValidationProtocol {
+	fn from(v3: v3::ValidationProtocol) -> Self {
+		VersionedValidationProtocol::V3(v3)
 	}
 }
 
@@ -317,7 +317,7 @@ macro_rules! impl_versioned_full_protocol_from {
 				match versioned_from {
 					Versioned::V1(x) => Versioned::V1(x.into()),
 					Versioned::V2(x) => Versioned::V2(x.into()),
-					Versioned::VStaging(x) => Versioned::VStaging(x.into()),
+					Versioned::V3(x) => Versioned::V3(x.into()),
 				}
 			}
 		}
@@ -331,7 +331,7 @@ macro_rules! impl_versioned_try_from {
 		$out:ty,
 		$v1_pat:pat => $v1_out:expr,
 		$v2_pat:pat => $v2_out:expr,
-		$vstaging_pat:pat => $vstaging_out:expr
+		$v3_pat:pat => $v3_out:expr
 	) => {
 		impl TryFrom<$from> for $out {
 			type Error = crate::WrongVariant;
@@ -341,7 +341,7 @@ macro_rules! impl_versioned_try_from {
 				match x {
 					Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)),
 					Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)),
-					Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)),
+					Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out)),
 					_ => Err(crate::WrongVariant),
 				}
 			}
@@ -355,8 +355,7 @@ macro_rules! impl_versioned_try_from {
 				match x {
 					Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())),
 					Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())),
-					Versioned::VStaging($vstaging_pat) =>
-						Ok(Versioned::VStaging($vstaging_out.clone())),
+					Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out.clone())),
 					_ => Err(crate::WrongVariant),
 				}
 			}
@@ -368,7 +367,7 @@ macro_rules! impl_versioned_try_from {
 pub type BitfieldDistributionMessage = Versioned<
 	v1::BitfieldDistributionMessage,
 	v2::BitfieldDistributionMessage,
-	vstaging::BitfieldDistributionMessage,
+	v3::BitfieldDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	BitfieldDistributionMessage,
@@ -380,14 +379,14 @@ impl_versioned_try_from!(
 	BitfieldDistributionMessage,
 	v1::ValidationProtocol::BitfieldDistribution(x) => x,
 	v2::ValidationProtocol::BitfieldDistribution(x) => x,
-	vstaging::ValidationProtocol::BitfieldDistribution(x) => x
+	v3::ValidationProtocol::BitfieldDistribution(x) => x
 );
 
 /// Version-annotated messages used by the statement distribution subsystem.
 pub type StatementDistributionMessage = Versioned<
 	v1::StatementDistributionMessage,
 	v2::StatementDistributionMessage,
-	vstaging::StatementDistributionMessage,
+	v3::StatementDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	StatementDistributionMessage,
@@ -399,14 +398,14 @@ impl_versioned_try_from!(
 	StatementDistributionMessage,
 	v1::ValidationProtocol::StatementDistribution(x) => x,
 	v2::ValidationProtocol::StatementDistribution(x) => x,
-	vstaging::ValidationProtocol::StatementDistribution(x) => x
+	v3::ValidationProtocol::StatementDistribution(x) => x
 );
 
 /// Version-annotated messages used by the approval distribution subsystem.
 pub type ApprovalDistributionMessage = Versioned<
 	v1::ApprovalDistributionMessage,
 	v2::ApprovalDistributionMessage,
-	vstaging::ApprovalDistributionMessage,
+	v3::ApprovalDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	ApprovalDistributionMessage,
@@ -418,7 +417,7 @@ impl_versioned_try_from!(
 	ApprovalDistributionMessage,
 	v1::ValidationProtocol::ApprovalDistribution(x) => x,
 	v2::ValidationProtocol::ApprovalDistribution(x) => x,
-	vstaging::ValidationProtocol::ApprovalDistribution(x) => x
+	v3::ValidationProtocol::ApprovalDistribution(x) => x
 
 );
 
@@ -426,7 +425,7 @@ impl_versioned_try_from!(
 pub type GossipSupportNetworkMessage = Versioned<
 	v1::GossipSupportNetworkMessage,
 	v2::GossipSupportNetworkMessage,
-	vstaging::GossipSupportNetworkMessage,
+	v3::GossipSupportNetworkMessage,
 >;
 
 // This is a void enum placeholder, so never gets sent over the wire.
@@ -871,19 +870,17 @@ pub mod v2 {
 	}
 }
 
-/// vstaging network protocol types, intended to become v3.
-/// Initial purpose is for chaning ApprovalDistributionMessage to
-/// include more than one assignment in the message.
-pub mod vstaging {
+/// v3 network protocol types.
+/// Purpose is for chaning ApprovalDistributionMessage to
+/// include more than one assignment and approval in a message.
+pub mod v3 {
 	use parity_scale_codec::{Decode, Encode};
 
-	use polkadot_node_primitives::approval::{
-		v1::IndirectSignedApprovalVote,
-		v2::{CandidateBitfield, IndirectAssignmentCertV2},
+	use polkadot_node_primitives::approval::v2::{
+		CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2,
 	};
 
-	/// This parts of the protocol did not change from v2, so just alias them in vstaging,
-	/// no reason why they can't be change untill vstaging becomes v3 and is released.
+	/// This parts of the protocol did not change from v2, so just alias them in v3.
 	pub use super::v2::{
 		declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest,
 		BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage,
@@ -903,7 +900,7 @@ pub mod vstaging {
 		Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>),
 		/// Approvals for candidates in some recent, unfinalized block.
 		#[codec(index = 1)]
-		Approvals(Vec<IndirectSignedApprovalVote>),
+		Approvals(Vec<IndirectSignedApprovalVoteV2>),
 	}
 
 	/// All network messages on the validation peer-set.