Approve multiple candidates with a single signature (#1191)

Initial implementation for the plan discussed here: https://github.com/paritytech/polkadot-sdk/issues/701
Built on top of https://github.com/paritytech/polkadot-sdk/pull/1178
v0: https://github.com/paritytech/polkadot/pull/7554,

## Overall idea

When approval-voting checks a candidate and is ready to advertise the
approval, defer it in a per-relay chain block until we either have
MAX_APPROVAL_COALESCE_COUNT candidates to sign or a candidate has stayed
MAX_APPROVALS_COALESCE_TICKS in the queue, in both cases we sign what
candidates we have available.

This should allow us to reduce the number of approvals messages we have
to create/send/verify. The parameters are configurable, so we should
find some values that balance:

- Security of the network: Delaying broadcasting of an approval
shouldn't but the finality at risk and to make sure that never happens
we won't delay sending a vote if we are past 2/3 from the no-show time.
- Scalability of the network: MAX_APPROVAL_COALESCE_COUNT = 1 &
MAX_APPROVALS_COALESCE_TICKS =0, is what we have now and we know from
the measurements we did on versi, it bottlenecks
approval-distribution/approval-voting when increase significantly the
number of validators and parachains
- Block storage: In case of disputes we have to import this votes on
chain and that increase the necessary storage with
MAX_APPROVAL_COALESCE_COUNT * CandidateHash per vote. Given that
disputes are not the normal way of the network functioning and we will
limit MAX_APPROVAL_COALESCE_COUNT in the single digits numbers, this
should be good enough. Alternatively, we could try to create a better
way to store this on-chain through indirection, if that's needed.

## Other fixes:
- Fixed the fact that we were sending random assignments to
non-validators, that was wrong because those won't do anything with it
and they won't gossip it either because they do not have a grid topology
set, so we would waste the random assignments.
- Added metrics to be able to debug potential no-shows and
mis-processing of approvals/assignments.

## TODO:
- [x] Get feedback, that this is moving in the right direction. @ordian
@sandreim @eskimor @burdges, let me know what you think.
- [x] More and more testing.
- [x]  Test in versi.
- [x] Make MAX_APPROVAL_COALESCE_COUNT &
MAX_APPROVAL_COALESCE_WAIT_MILLIS a parachain host configuration.
- [x] Make sure the backwards compatibility works correctly
- [x] Make sure this direction is compatible with other streams of work:
https://github.com/paritytech/polkadot-sdk/issues/635 &
https://github.com/paritytech/polkadot-sdk/issues/742
- [x] Final versi burn-in before merging

---------

Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>

polkadot/node/network/protocol/src/grid_topology.rs

@@ -73,12 +73,20 @@ pub struct SessionGridTopology {
 	shuffled_indices: Vec<usize>,
 	/// The canonical shuffling of validators for the session.
 	canonical_shuffling: Vec<TopologyPeerInfo>,
+	/// The list of peer-ids in an efficient way to search.
+	peer_ids: HashSet<PeerId>,
 }
 
 impl SessionGridTopology {
 	/// Create a new session grid topology.
 	pub fn new(shuffled_indices: Vec<usize>, canonical_shuffling: Vec<TopologyPeerInfo>) -> Self {
-		SessionGridTopology { shuffled_indices, canonical_shuffling }
+		let mut peer_ids = HashSet::new();
+		for peer_info in canonical_shuffling.iter() {
+			for peer_id in peer_info.peer_ids.iter() {
+				peer_ids.insert(*peer_id);
+			}
+		}
+		SessionGridTopology { shuffled_indices, canonical_shuffling, peer_ids }
 	}
 
 	/// Produces the outgoing routing logic for a particular peer.
@@ -111,6 +119,11 @@ impl SessionGridTopology {
 
 		Some(grid_subset)
 	}
+
+	/// Tells if a given peer id is validator in a session
+	pub fn is_validator(&self, peer: &PeerId) -> bool {
+		self.peer_ids.contains(peer)
+	}
 }
 
 struct MatrixNeighbors<R, C> {
@@ -273,6 +286,11 @@ impl SessionGridTopologyEntry {
 	pub fn get(&self) -> &SessionGridTopology {
 		&self.topology
 	}
+
+	/// Tells if a given peer id is validator in a session
+	pub fn is_validator(&self, peer: &PeerId) -> bool {
+		self.topology.is_validator(peer)
+	}
 }
 
 /// A set of topologies indexed by session
@@ -347,6 +365,7 @@ impl Default for SessionBoundGridTopologyStorage {
 					topology: SessionGridTopology {
 						shuffled_indices: Vec::new(),
 						canonical_shuffling: Vec::new(),
+						peer_ids: Default::default(),
 					},
 					local_neighbors: GridNeighbors::empty(),
 				},

polkadot/node/network/protocol/src/lib.rs

@@ -253,29 +253,29 @@ impl View {
 
 /// A protocol-versioned type.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Versioned<V1, V2, VStaging = V2> {
+pub enum Versioned<V1, V2, V3 = V2> {
 	/// V1 type.
 	V1(V1),
 	/// V2 type.
 	V2(V2),
-	/// VStaging type
-	VStaging(VStaging),
+	/// V3 type
+	V3(V3),
 }
 
-impl<V1: Clone, V2: Clone, VStaging: Clone> Versioned<&'_ V1, &'_ V2, &'_ VStaging> {
+impl<V1: Clone, V2: Clone, V3: Clone> Versioned<&'_ V1, &'_ V2, &'_ V3> {
 	/// Convert to a fully-owned version of the message.
-	pub fn clone_inner(&self) -> Versioned<V1, V2, VStaging> {
+	pub fn clone_inner(&self) -> Versioned<V1, V2, V3> {
 		match *self {
 			Versioned::V1(inner) => Versioned::V1(inner.clone()),
 			Versioned::V2(inner) => Versioned::V2(inner.clone()),
-			Versioned::VStaging(inner) => Versioned::VStaging(inner.clone()),
+			Versioned::V3(inner) => Versioned::V3(inner.clone()),
 		}
 	}
 }
 
 /// All supported versions of the validation protocol message.
 pub type VersionedValidationProtocol =
-	Versioned<v1::ValidationProtocol, v2::ValidationProtocol, vstaging::ValidationProtocol>;
+	Versioned<v1::ValidationProtocol, v2::ValidationProtocol, v3::ValidationProtocol>;
 
 impl From<v1::ValidationProtocol> for VersionedValidationProtocol {
 	fn from(v1: v1::ValidationProtocol) -> Self {
@@ -289,9 +289,9 @@ impl From<v2::ValidationProtocol> for VersionedValidationProtocol {
 	}
 }
 
-impl From<vstaging::ValidationProtocol> for VersionedValidationProtocol {
-	fn from(vstaging: vstaging::ValidationProtocol) -> Self {
-		VersionedValidationProtocol::VStaging(vstaging)
+impl From<v3::ValidationProtocol> for VersionedValidationProtocol {
+	fn from(v3: v3::ValidationProtocol) -> Self {
+		VersionedValidationProtocol::V3(v3)
 	}
 }
 
@@ -317,7 +317,7 @@ macro_rules! impl_versioned_full_protocol_from {
 				match versioned_from {
 					Versioned::V1(x) => Versioned::V1(x.into()),
 					Versioned::V2(x) => Versioned::V2(x.into()),
-					Versioned::VStaging(x) => Versioned::VStaging(x.into()),
+					Versioned::V3(x) => Versioned::V3(x.into()),
 				}
 			}
 		}
@@ -331,7 +331,7 @@ macro_rules! impl_versioned_try_from {
 		$out:ty,
 		$v1_pat:pat => $v1_out:expr,
 		$v2_pat:pat => $v2_out:expr,
-		$vstaging_pat:pat => $vstaging_out:expr
+		$v3_pat:pat => $v3_out:expr
 	) => {
 		impl TryFrom<$from> for $out {
 			type Error = crate::WrongVariant;
@@ -341,7 +341,7 @@ macro_rules! impl_versioned_try_from {
 				match x {
 					Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out)),
 					Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out)),
-					Versioned::VStaging($vstaging_pat) => Ok(Versioned::VStaging($vstaging_out)),
+					Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out)),
 					_ => Err(crate::WrongVariant),
 				}
 			}
@@ -355,8 +355,7 @@ macro_rules! impl_versioned_try_from {
 				match x {
 					Versioned::V1($v1_pat) => Ok(Versioned::V1($v1_out.clone())),
 					Versioned::V2($v2_pat) => Ok(Versioned::V2($v2_out.clone())),
-					Versioned::VStaging($vstaging_pat) =>
-						Ok(Versioned::VStaging($vstaging_out.clone())),
+					Versioned::V3($v3_pat) => Ok(Versioned::V3($v3_out.clone())),
 					_ => Err(crate::WrongVariant),
 				}
 			}
@@ -368,7 +367,7 @@ macro_rules! impl_versioned_try_from {
 pub type BitfieldDistributionMessage = Versioned<
 	v1::BitfieldDistributionMessage,
 	v2::BitfieldDistributionMessage,
-	vstaging::BitfieldDistributionMessage,
+	v3::BitfieldDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	BitfieldDistributionMessage,
@@ -380,14 +379,14 @@ impl_versioned_try_from!(
 	BitfieldDistributionMessage,
 	v1::ValidationProtocol::BitfieldDistribution(x) => x,
 	v2::ValidationProtocol::BitfieldDistribution(x) => x,
-	vstaging::ValidationProtocol::BitfieldDistribution(x) => x
+	v3::ValidationProtocol::BitfieldDistribution(x) => x
 );
 
 /// Version-annotated messages used by the statement distribution subsystem.
 pub type StatementDistributionMessage = Versioned<
 	v1::StatementDistributionMessage,
 	v2::StatementDistributionMessage,
-	vstaging::StatementDistributionMessage,
+	v3::StatementDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	StatementDistributionMessage,
@@ -399,14 +398,14 @@ impl_versioned_try_from!(
 	StatementDistributionMessage,
 	v1::ValidationProtocol::StatementDistribution(x) => x,
 	v2::ValidationProtocol::StatementDistribution(x) => x,
-	vstaging::ValidationProtocol::StatementDistribution(x) => x
+	v3::ValidationProtocol::StatementDistribution(x) => x
 );
 
 /// Version-annotated messages used by the approval distribution subsystem.
 pub type ApprovalDistributionMessage = Versioned<
 	v1::ApprovalDistributionMessage,
 	v2::ApprovalDistributionMessage,
-	vstaging::ApprovalDistributionMessage,
+	v3::ApprovalDistributionMessage,
 >;
 impl_versioned_full_protocol_from!(
 	ApprovalDistributionMessage,
@@ -418,7 +417,7 @@ impl_versioned_try_from!(
 	ApprovalDistributionMessage,
 	v1::ValidationProtocol::ApprovalDistribution(x) => x,
 	v2::ValidationProtocol::ApprovalDistribution(x) => x,
-	vstaging::ValidationProtocol::ApprovalDistribution(x) => x
+	v3::ValidationProtocol::ApprovalDistribution(x) => x
 
 );
 
@@ -426,7 +425,7 @@ impl_versioned_try_from!(
 pub type GossipSupportNetworkMessage = Versioned<
 	v1::GossipSupportNetworkMessage,
 	v2::GossipSupportNetworkMessage,
-	vstaging::GossipSupportNetworkMessage,
+	v3::GossipSupportNetworkMessage,
 >;
 
 // This is a void enum placeholder, so never gets sent over the wire.
@@ -871,19 +870,17 @@ pub mod v2 {
 	}
 }
 
-/// vstaging network protocol types, intended to become v3.
-/// Initial purpose is for chaning ApprovalDistributionMessage to
-/// include more than one assignment in the message.
-pub mod vstaging {
+/// v3 network protocol types.
+/// Purpose is for chaning ApprovalDistributionMessage to
+/// include more than one assignment and approval in a message.
+pub mod v3 {
 	use parity_scale_codec::{Decode, Encode};
 
-	use polkadot_node_primitives::approval::{
-		v1::IndirectSignedApprovalVote,
-		v2::{CandidateBitfield, IndirectAssignmentCertV2},
+	use polkadot_node_primitives::approval::v2::{
+		CandidateBitfield, IndirectAssignmentCertV2, IndirectSignedApprovalVoteV2,
 	};
 
-	/// This parts of the protocol did not change from v2, so just alias them in vstaging,
-	/// no reason why they can't be change untill vstaging becomes v3 and is released.
+	/// This parts of the protocol did not change from v2, so just alias them in v3.
 	pub use super::v2::{
 		declare_signature_payload, BackedCandidateAcknowledgement, BackedCandidateManifest,
 		BitfieldDistributionMessage, GossipSupportNetworkMessage, StatementDistributionMessage,
@@ -903,7 +900,7 @@ pub mod vstaging {
 		Assignments(Vec<(IndirectAssignmentCertV2, CandidateBitfield)>),
 		/// Approvals for candidates in some recent, unfinalized block.
 		#[codec(index = 1)]
-		Approvals(Vec<IndirectSignedApprovalVote>),
+		Approvals(Vec<IndirectSignedApprovalVoteV2>),
 	}
 
 	/// All network messages on the validation peer-set.

polkadot/node/network/protocol/src/peer_set.rs

@@ -73,7 +73,11 @@ impl PeerSet {
 		// Networking layer relies on `get_main_name()` being the main name of the protocol
 		// for peersets and connection management.
 		let protocol = peerset_protocol_names.get_main_name(self);
-		let fallback_names = PeerSetProtocolNames::get_fallback_names(self);
+		let fallback_names = PeerSetProtocolNames::get_fallback_names(
+			self,
+			&peerset_protocol_names.genesis_hash,
+			peerset_protocol_names.fork_id.as_deref(),
+		);
 		let max_notification_size = self.get_max_notification_size(is_authority);
 
 		match self {
@@ -127,15 +131,8 @@ impl PeerSet {
 	/// Networking layer relies on `get_main_version()` being the version
 	/// of the main protocol name reported by [`PeerSetProtocolNames::get_main_name()`].
 	pub fn get_main_version(self) -> ProtocolVersion {
-		#[cfg(not(feature = "network-protocol-staging"))]
 		match self {
-			PeerSet::Validation => ValidationVersion::V2.into(),
-			PeerSet::Collation => CollationVersion::V2.into(),
-		}
-
-		#[cfg(feature = "network-protocol-staging")]
-		match self {
-			PeerSet::Validation => ValidationVersion::VStaging.into(),
+			PeerSet::Validation => ValidationVersion::V3.into(),
 			PeerSet::Collation => CollationVersion::V2.into(),
 		}
 	}
@@ -163,7 +160,7 @@ impl PeerSet {
 					Some("validation/1")
 				} else if version == ValidationVersion::V2.into() {
 					Some("validation/2")
-				} else if version == ValidationVersion::VStaging.into() {
+				} else if version == ValidationVersion::V3.into() {
 					Some("validation/3")
 				} else {
 					None
@@ -236,9 +233,10 @@ pub enum ValidationVersion {
 	V1 = 1,
 	/// The second version.
 	V2 = 2,
-	/// The staging version to gather changes
-	/// that before the release become v3.
-	VStaging = 3,
+	/// The third version where changes to ApprovalDistributionMessage had been made.
+	/// The changes are translatable to V2 format untill assignments v2 and approvals
+	/// coalescing is enabled through a runtime upgrade.
+	V3 = 3,
 }
 
 /// Supported collation protocol versions. Only versions defined here must be used in the codebase.
@@ -299,6 +297,8 @@ impl From<CollationVersion> for ProtocolVersion {
 pub struct PeerSetProtocolNames {
 	protocols: HashMap<ProtocolName, (PeerSet, ProtocolVersion)>,
 	names: HashMap<(PeerSet, ProtocolVersion), ProtocolName>,
+	genesis_hash: Hash,
+	fork_id: Option<String>,
 }
 
 impl PeerSetProtocolNames {
@@ -333,7 +333,7 @@ impl PeerSetProtocolNames {
 			}
 			Self::register_legacy_protocol(&mut protocols, protocol);
 		}
-		Self { protocols, names }
+		Self { protocols, names, genesis_hash, fork_id: fork_id.map(|fork_id| fork_id.into()) }
 	}
 
 	/// Helper function to register main protocol.
@@ -437,9 +437,30 @@ impl PeerSetProtocolNames {
 	}
 
 	/// Get the protocol fallback names. Currently only holds the legacy name
-	/// for `LEGACY_PROTOCOL_VERSION` = 1.
-	fn get_fallback_names(protocol: PeerSet) -> Vec<ProtocolName> {
-		std::iter::once(Self::get_legacy_name(protocol)).collect()
+	/// for `LEGACY_PROTOCOL_VERSION` = 1 and v2 for validation.
+	fn get_fallback_names(
+		protocol: PeerSet,
+		genesis_hash: &Hash,
+		fork_id: Option<&str>,
+	) -> Vec<ProtocolName> {
+		let mut fallbacks = vec![Self::get_legacy_name(protocol)];
+		match protocol {
+			PeerSet::Validation => {
+				// Fallbacks are tried one by one, till one matches so push v2 at the top, so
+				// that it is used ahead of the legacy one(v1).
+				fallbacks.insert(
+					0,
+					Self::generate_name(
+						genesis_hash,
+						fork_id,
+						protocol,
+						ValidationVersion::V2.into(),
+					),
+				)
+			},
+			PeerSet::Collation => {},
+		};
+		fallbacks
 	}
 }