Approve multiple candidates with a single signature (#1191)

Initial implementation for the plan discussed here: https://github.com/paritytech/polkadot-sdk/issues/701
Built on top of https://github.com/paritytech/polkadot-sdk/pull/1178
v0: https://github.com/paritytech/polkadot/pull/7554,

## Overall idea

When approval-voting checks a candidate and is ready to advertise the
approval, defer it in a per-relay chain block until we either have
MAX_APPROVAL_COALESCE_COUNT candidates to sign or a candidate has stayed
MAX_APPROVALS_COALESCE_TICKS in the queue, in both cases we sign what
candidates we have available.

This should allow us to reduce the number of approvals messages we have
to create/send/verify. The parameters are configurable, so we should
find some values that balance:

- Security of the network: Delaying broadcasting of an approval
shouldn't but the finality at risk and to make sure that never happens
we won't delay sending a vote if we are past 2/3 from the no-show time.
- Scalability of the network: MAX_APPROVAL_COALESCE_COUNT = 1 &
MAX_APPROVALS_COALESCE_TICKS =0, is what we have now and we know from
the measurements we did on versi, it bottlenecks
approval-distribution/approval-voting when increase significantly the
number of validators and parachains
- Block storage: In case of disputes we have to import this votes on
chain and that increase the necessary storage with
MAX_APPROVAL_COALESCE_COUNT * CandidateHash per vote. Given that
disputes are not the normal way of the network functioning and we will
limit MAX_APPROVAL_COALESCE_COUNT in the single digits numbers, this
should be good enough. Alternatively, we could try to create a better
way to store this on-chain through indirection, if that's needed.

## Other fixes:
- Fixed the fact that we were sending random assignments to
non-validators, that was wrong because those won't do anything with it
and they won't gossip it either because they do not have a grid topology
set, so we would waste the random assignments.
- Added metrics to be able to debug potential no-shows and
mis-processing of approvals/assignments.

## TODO:
- [x] Get feedback, that this is moving in the right direction. @ordian
@sandreim @eskimor @burdges, let me know what you think.
- [x] More and more testing.
- [x]  Test in versi.
- [x] Make MAX_APPROVAL_COALESCE_COUNT &
MAX_APPROVAL_COALESCE_WAIT_MILLIS a parachain host configuration.
- [x] Make sure the backwards compatibility works correctly
- [x] Make sure this direction is compatible with other streams of work:
https://github.com/paritytech/polkadot-sdk/issues/635 &
https://github.com/paritytech/polkadot-sdk/issues/742
- [x] Final versi burn-in before merging

---------

Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io>

polkadot/node/network/bridge/src/network.rs

@@ -33,7 +33,7 @@ use sc_network::{
 use polkadot_node_network_protocol::{
 	peer_set::{CollationVersion, PeerSet, ProtocolVersion, ValidationVersion},
 	request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests},
-	v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, PeerId,
+	v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, PeerId,
 };
 use polkadot_primitives::{AuthorityDiscoveryId, Block, Hash};
 
@@ -62,20 +62,20 @@ pub(crate) fn send_validation_message_v1(
 	);
 }
 
-// Helper function to send a validation vstaging message to a list of peers.
+// Helper function to send a validation v3 message to a list of peers.
 // Messages are always sent via the main protocol, even legacy protocol messages.
-pub(crate) fn send_validation_message_vstaging(
+pub(crate) fn send_validation_message_v3(
 	peers: Vec<PeerId>,
-	message: WireMessage<protocol_vstaging::ValidationProtocol>,
+	message: WireMessage<protocol_v3::ValidationProtocol>,
 	metrics: &Metrics,
 	notification_sinks: &Arc<Mutex<HashMap<(PeerSet, PeerId), Box<dyn MessageSink>>>>,
 ) {
-	gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation vstaging message to peers",);
+	gum::trace!(target: LOG_TARGET, ?peers, ?message, "Sending validation v3 message to peers",);
 
 	send_message(
 		peers,
 		PeerSet::Validation,
-		ValidationVersion::VStaging.into(),
+		ValidationVersion::V3.into(),
 		message,
 		metrics,
 		notification_sinks,

polkadot/node/network/bridge/src/rx/mod.rs

@@ -37,8 +37,8 @@ use polkadot_node_network_protocol::{
 		CollationVersion, PeerSet, PeerSetProtocolNames, PerPeerSet, ProtocolVersion,
 		ValidationVersion,
 	},
-	v1 as protocol_v1, v2 as protocol_v2, vstaging as protocol_vstaging, ObservedRole, OurView,
-	PeerId, UnifiedReputationChange as Rep, View,
+	v1 as protocol_v1, v2 as protocol_v2, v3 as protocol_v3, ObservedRole, OurView, PeerId,
+	UnifiedReputationChange as Rep, View,
 };
 
 use polkadot_node_subsystem::{
@@ -70,7 +70,7 @@ use super::validator_discovery;
 /// Defines the `Network` trait with an implementation for an `Arc<NetworkService>`.
 use crate::network::{
 	send_collation_message_v1, send_collation_message_v2, send_validation_message_v1,
-	send_validation_message_v2, send_validation_message_vstaging, Network,
+	send_validation_message_v2, send_validation_message_v3, Network,
 };
 use crate::{network::get_peer_id_by_authority_id, WireMessage};
 
@@ -294,9 +294,9 @@ async fn handle_validation_message<AD>(
 					metrics,
 					notification_sinks,
 				),
-				ValidationVersion::VStaging => send_validation_message_vstaging(
+				ValidationVersion::V3 => send_validation_message_v3(
 					vec![peer],
-					WireMessage::<protocol_vstaging::ValidationProtocol>::ViewUpdate(local_view),
+					WireMessage::<protocol_v3::ValidationProtocol>::ViewUpdate(local_view),
 					metrics,
 					notification_sinks,
 				),
@@ -360,48 +360,47 @@ async fn handle_validation_message<AD>(
 				?peer,
 			);
 
-			let (events, reports) =
-				if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) {
-					handle_peer_messages::<protocol_v1::ValidationProtocol, _>(
-						peer,
-						PeerSet::Validation,
-						&mut shared.0.lock().validation_peers,
-						vec![notification.into()],
-						metrics,
-					)
-				} else if expected_versions[PeerSet::Validation] ==
-					Some(ValidationVersion::V2.into())
-				{
-					handle_peer_messages::<protocol_v2::ValidationProtocol, _>(
-						peer,
-						PeerSet::Validation,
-						&mut shared.0.lock().validation_peers,
-						vec![notification.into()],
-						metrics,
-					)
-				} else if expected_versions[PeerSet::Validation] ==
-					Some(ValidationVersion::VStaging.into())
-				{
-					handle_peer_messages::<protocol_vstaging::ValidationProtocol, _>(
-						peer,
-						PeerSet::Validation,
-						&mut shared.0.lock().validation_peers,
-						vec![notification.into()],
-						metrics,
-					)
-				} else {
-					gum::warn!(
-						target: LOG_TARGET,
-						version = ?expected_versions[PeerSet::Validation],
-						"Major logic bug. Peer somehow has unsupported validation protocol version."
-					);
+			let (events, reports) = if expected_versions[PeerSet::Validation] ==
+				Some(ValidationVersion::V1.into())
+			{
+				handle_peer_messages::<protocol_v1::ValidationProtocol, _>(
+					peer,
+					PeerSet::Validation,
+					&mut shared.0.lock().validation_peers,
+					vec![notification.into()],
+					metrics,
+				)
+			} else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V2.into()) {
+				handle_peer_messages::<protocol_v2::ValidationProtocol, _>(
+					peer,
+					PeerSet::Validation,
+					&mut shared.0.lock().validation_peers,
+					vec![notification.into()],
+					metrics,
+				)
+			} else if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V3.into()) {
+				handle_peer_messages::<protocol_v3::ValidationProtocol, _>(
+					peer,
+					PeerSet::Validation,
+					&mut shared.0.lock().validation_peers,
+					vec![notification.into()],
+					metrics,
+				)
+			} else {
+				gum::warn!(
+					target: LOG_TARGET,
+					version = ?expected_versions[PeerSet::Validation],
+					"Major logic bug. Peer somehow has unsupported validation protocol version."
+				);
 
-					never!("Only versions 1 and 2 are supported; peer set connection checked above; qed");
+				never!(
+					"Only versions 1 and 2 are supported; peer set connection checked above; qed"
+				);
 
-					// If a peer somehow triggers this, we'll disconnect them
-					// eventually.
-					(Vec::new(), vec![UNCONNECTED_PEERSET_COST])
-				};
+				// If a peer somehow triggers this, we'll disconnect them
+				// eventually.
+				(Vec::new(), vec![UNCONNECTED_PEERSET_COST])
+			};
 
 			for report in reports {
 				network_service.report_peer(peer, report.into());
@@ -980,8 +979,8 @@ fn update_our_view<Context>(
 		filter_by_peer_version(&validation_peers, ValidationVersion::V2.into());
 	let v2_collation_peers = filter_by_peer_version(&collation_peers, CollationVersion::V2.into());
 
-	let vstaging_validation_peers =
-		filter_by_peer_version(&validation_peers, ValidationVersion::VStaging.into());
+	let v3_validation_peers =
+		filter_by_peer_version(&validation_peers, ValidationVersion::V3.into());
 
 	send_validation_message_v1(
 		v1_validation_peers,
@@ -1011,8 +1010,8 @@ fn update_our_view<Context>(
 		notification_sinks,
 	);
 
-	send_validation_message_vstaging(
-		vstaging_validation_peers,
+	send_validation_message_v3(
+		v3_validation_peers,
 		WireMessage::ViewUpdate(new_view.clone()),
 		metrics,
 		notification_sinks,

polkadot/node/network/bridge/src/rx/tests.rs

@@ -224,7 +224,7 @@ impl TestNetworkHandle {
 				PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/1")),
 				PeerSet::Collation => Some(ProtocolName::from("/polkadot/collation/1")),
 			},
-			ValidationVersion::VStaging => match peer_set {
+			ValidationVersion::V3 => match peer_set {
 				PeerSet::Validation => Some(ProtocolName::from("/polkadot/validation/3")),
 				PeerSet::Collation => unreachable!(),
 			},
@@ -1433,8 +1433,8 @@ fn network_protocol_versioning_view_update() {
 				ValidationVersion::V2 =>
 					WireMessage::<protocol_v2::ValidationProtocol>::ViewUpdate(view.clone())
 						.encode(),
-				ValidationVersion::VStaging =>
-					WireMessage::<protocol_vstaging::ValidationProtocol>::ViewUpdate(view.clone())
+				ValidationVersion::V3 =>
+					WireMessage::<protocol_v3::ValidationProtocol>::ViewUpdate(view.clone())
 						.encode(),
 			};
 			assert_network_actions_contains(
@@ -1469,7 +1469,7 @@ fn network_protocol_versioning_subsystem_msg() {
 				NetworkBridgeEvent::PeerConnected(
 					peer,
 					ObservedRole::Full,
-					ValidationVersion::V2.into(),
+					ValidationVersion::V3.into(),
 					None,
 				),
 				&mut virtual_overseer,
@@ -1484,9 +1484,9 @@ fn network_protocol_versioning_subsystem_msg() {
 		}
 
 		let approval_distribution_message =
-			protocol_v2::ApprovalDistributionMessage::Approvals(Vec::new());
+			protocol_v3::ApprovalDistributionMessage::Approvals(Vec::new());
 
-		let msg = protocol_v2::ValidationProtocol::ApprovalDistribution(
+		let msg = protocol_v3::ValidationProtocol::ApprovalDistribution(
 			approval_distribution_message.clone(),
 		);
 
@@ -1502,7 +1502,7 @@ fn network_protocol_versioning_subsystem_msg() {
 			virtual_overseer.recv().await,
 			AllMessages::ApprovalDistribution(
 				ApprovalDistributionMessage::NetworkBridgeUpdate(
-					NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m))
+					NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m))
 				)
 			) => {
 				assert_eq!(p, peer);
@@ -1536,7 +1536,7 @@ fn network_protocol_versioning_subsystem_msg() {
 			virtual_overseer.recv().await,
 			AllMessages::StatementDistribution(
 				StatementDistributionMessage::NetworkBridgeUpdate(
-					NetworkBridgeEvent::PeerMessage(p, Versioned::V2(m))
+					NetworkBridgeEvent::PeerMessage(p, Versioned::V3(m))
 				)
 			) => {
 				assert_eq!(p, peer);

polkadot/node/network/bridge/src/tx/mod.rs

@@ -41,7 +41,7 @@ use crate::validator_discovery;
 /// Defines the `Network` trait with an implementation for an `Arc<NetworkService>`.
 use crate::network::{
 	send_collation_message_v1, send_collation_message_v2, send_validation_message_v1,
-	send_validation_message_v2, send_validation_message_vstaging, Network,
+	send_validation_message_v2, send_validation_message_v3, Network,
 };
 
 use crate::metrics::Metrics;
@@ -205,7 +205,7 @@ where
 					&metrics,
 					notification_sinks,
 				),
-				Versioned::VStaging(msg) => send_validation_message_vstaging(
+				Versioned::V3(msg) => send_validation_message_v3(
 					peers,
 					WireMessage::ProtocolMessage(msg),
 					&metrics,
@@ -235,7 +235,7 @@ where
 						&metrics,
 						notification_sinks,
 					),
-					Versioned::VStaging(msg) => send_validation_message_vstaging(
+					Versioned::V3(msg) => send_validation_message_v3(
 						peers,
 						WireMessage::ProtocolMessage(msg),
 						&metrics,
@@ -264,7 +264,7 @@ where
 					&metrics,
 					notification_sinks,
 				),
-				Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2(
+				Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2(
 					peers,
 					WireMessage::ProtocolMessage(msg),
 					&metrics,
@@ -287,7 +287,7 @@ where
 						&metrics,
 						notification_sinks,
 					),
-					Versioned::V2(msg) | Versioned::VStaging(msg) => send_collation_message_v2(
+					Versioned::V2(msg) | Versioned::V3(msg) => send_collation_message_v2(
 						peers,
 						WireMessage::ProtocolMessage(msg),
 						&metrics,