Allow to broadcast network messages in parallel (#1409)

This PR addresses multiple issues pending:

* [x] Update orchestra to the recent version and test how the node
performs
* [x] Add some useful metrics for outbound network bridge
* [x] Try to send incoming network requests to all subsystems without
blocking on some particular subsystem in that loop
* [x] Fix all incompatibilities between orchestra and polkadot code
(e.g. malus node)

polkadot/node/network/bridge/src/rx/mod.rs

@@ -20,7 +20,10 @@ use super::*;
 
 use always_assert::never;
 use bytes::Bytes;
-use futures::stream::BoxStream;
+use futures::{
+	future::BoxFuture,
+	stream::{BoxStream, FuturesUnordered, StreamExt},
+};
 use parity_scale_codec::{Decode, DecodeAll};
 
 use sc_network::Event as NetworkEvent;
@@ -244,6 +247,7 @@ where
 								NetworkBridgeEvent::PeerViewChange(peer, View::default()),
 							],
 							&mut sender,
+							&metrics,
 						)
 						.await;
 
@@ -352,6 +356,7 @@ where
 							dispatch_validation_event_to_all(
 								NetworkBridgeEvent::PeerDisconnected(peer),
 								&mut sender,
+								&metrics,
 							)
 							.await,
 						PeerSet::Collation =>
@@ -490,7 +495,7 @@ where
 						network_service.report_peer(remote, report.into());
 					}
 
-					dispatch_validation_events_to_all(events, &mut sender).await;
+					dispatch_validation_events_to_all(events, &mut sender, &metrics).await;
 				}
 
 				if !c_messages.is_empty() {
@@ -992,8 +997,9 @@ fn send_collation_message_vstaging(
 async fn dispatch_validation_event_to_all(
 	event: NetworkBridgeEvent<net_protocol::VersionedValidationProtocol>,
 	ctx: &mut impl overseer::NetworkBridgeRxSenderTrait,
+	metrics: &Metrics,
 ) {
-	dispatch_validation_events_to_all(std::iter::once(event), ctx).await
+	dispatch_validation_events_to_all(std::iter::once(event), ctx, metrics).await
 }
 
 async fn dispatch_collation_event_to_all(
@@ -1038,20 +1044,65 @@ fn dispatch_collation_event_to_all_unbounded(
 	}
 }
 
+fn send_or_queue_validation_event<E, Sender>(
+	event: E,
+	sender: &mut Sender,
+	delayed_queue: &FuturesUnordered<BoxFuture<'static, ()>>,
+) where
+	E: Send + 'static,
+	Sender: overseer::NetworkBridgeRxSenderTrait + overseer::SubsystemSender<E>,
+{
+	match sender.try_send_message(event) {
+		Ok(()) => {},
+		Err(overseer::TrySendError::Full(event)) => {
+			let mut sender = sender.clone();
+			delayed_queue.push(Box::pin(async move {
+				sender.send_message(event).await;
+			}));
+		},
+		Err(overseer::TrySendError::Closed(_)) => {
+			panic!(
+				"NetworkBridgeRxSender is closed when trying to send event of type: {}",
+				std::any::type_name::<E>()
+			);
+		},
+	}
+}
+
 async fn dispatch_validation_events_to_all<I>(
 	events: I,
 	sender: &mut impl overseer::NetworkBridgeRxSenderTrait,
+	metrics: &Metrics,
 ) where
 	I: IntoIterator<Item = NetworkBridgeEvent<net_protocol::VersionedValidationProtocol>>,
 	I::IntoIter: Send,
 {
+	let delayed_messages: FuturesUnordered<BoxFuture<'static, ()>> = FuturesUnordered::new();
+
+	// Fast path for sending events to subsystems, if any subsystem's queue is full, we hold
+	// the slow path future in the `delayed_messages` queue.
 	for event in events {
-		sender
-			.send_messages(event.focus().map(StatementDistributionMessage::from))
-			.await;
-		sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await;
-		sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await;
-		sender.send_messages(event.focus().map(GossipSupportMessage::from)).await;
+		if let Ok(msg) = event.focus().map(StatementDistributionMessage::from) {
+			send_or_queue_validation_event(msg, sender, &delayed_messages);
+		}
+		if let Ok(msg) = event.focus().map(BitfieldDistributionMessage::from) {
+			send_or_queue_validation_event(msg, sender, &delayed_messages);
+		}
+		if let Ok(msg) = event.focus().map(ApprovalDistributionMessage::from) {
+			send_or_queue_validation_event(msg, sender, &delayed_messages);
+		}
+		if let Ok(msg) = event.focus().map(GossipSupportMessage::from) {
+			send_or_queue_validation_event(msg, sender, &delayed_messages);
+		}
+	}
+
+	let delayed_messages_count = delayed_messages.len();
+	metrics.on_delayed_rx_queue(delayed_messages_count);
+
+	if delayed_messages_count > 0 {
+		// Here we wait for all the delayed messages to be sent.
+		let _timer = metrics.time_delayed_rx_events(); // Dropped after `await` is completed
+		let _: Vec<()> = delayed_messages.collect().await;
 	}
 }