Batch gossip messages (#4055)

substrate/core/network/src/protocol.rs

@@ -69,12 +69,14 @@ const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100);
 const PROPAGATE_TIMEOUT: time::Duration = time::Duration::from_millis(2900);
 
 /// Current protocol version.
-pub(crate) const CURRENT_VERSION: u32 = 4;
+pub(crate) const CURRENT_VERSION: u32 = 5;
 /// Lowest version we support
 pub(crate) const MIN_VERSION: u32 = 3;
 
 // Maximum allowed entries in `BlockResponse`
 const MAX_BLOCK_DATA_RESPONSE: u32 = 128;
+// Maximum allowed entries in `ConsensusBatch`
+const MAX_CONSENSUS_MESSAGES: usize = 256;
 /// When light node connects to the full node and the full node is behind light node
 /// for at least `LIGHT_MAXIMAL_BLOCKS_DIFFERENCE` blocks, we consider it unuseful
 /// and disconnect to free connection slot.
@@ -298,7 +300,7 @@ pub trait Context<B: BlockT> {
 	fn disconnect_peer(&mut self, who: PeerId);
 
 	/// Send a consensus message to a peer.
-	fn send_consensus(&mut self, who: PeerId, consensus: ConsensusMessage);
+	fn send_consensus(&mut self, who: PeerId, messages: Vec<ConsensusMessage>);
 
 	/// Send a chain-specific message to a peer.
 	fn send_chain_specific(&mut self, who: PeerId, message: Vec<u8>);
@@ -330,14 +332,34 @@ impl<'a, B: BlockT + 'a, H: ExHashT + 'a> Context<B> for ProtocolContext<'a, B,
 		self.behaviour.disconnect_peer(&who)
 	}
 
-	fn send_consensus(&mut self, who: PeerId, consensus: ConsensusMessage) {
+	fn send_consensus(&mut self, who: PeerId, messages: Vec<ConsensusMessage>) {
+		if self.context_data.peers.get(&who).map_or(false, |peer| peer.info.protocol_version > 4) {
+			let mut batch = Vec::new();
+			let len = messages.len();
+			for (index, message) in messages.into_iter().enumerate() {
+				batch.reserve(MAX_CONSENSUS_MESSAGES);
+				batch.push(message);
+				if batch.len() == MAX_CONSENSUS_MESSAGES || index == len - 1 {
 					send_message::<B> (
 						self.behaviour,
 						&mut self.context_data.stats,
 						&who,
-			GenericMessage::Consensus(consensus)
+						GenericMessage::ConsensusBatch(std::mem::replace(&mut batch, Vec::new())),
 					)
 				}
+			}
+		} else {
+			// Backwards compatibility
+			for message in messages {
+				send_message::<B> (
+					self.behaviour,
+					&mut self.context_data.stats,
+					&who,
+					GenericMessage::Consensus(message)
+				)
+			}
+		}
+	}
 
 	fn send_chain_specific(&mut self, who: PeerId, message: Vec<u8>) {
 		send_message::<B> (
@@ -598,13 +620,18 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 			GenericMessage::RemoteReadChildRequest(request) =>
 				self.on_remote_read_child_request(who, request),
 			GenericMessage::Consensus(msg) => {
-				if self.context_data.peers.get(&who).map_or(false, |peer| peer.info.protocol_version > 2) {
 				self.consensus_gossip.on_incoming(
 					&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),
 					who,
-						msg,
+					vec![msg],
 				);
 			}
+			GenericMessage::ConsensusBatch(messages) => {
+				self.consensus_gossip.on_incoming(
+					&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),
+					who,
+					messages,
+				);
 			}
 			GenericMessage::ChainSpecific(msg) => self.specialization.on_message(
 				&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),

substrate/core/network/src/protocol/consensus_gossip.rs

@@ -171,10 +171,10 @@ impl<'g, 'p, B: BlockT> ValidatorContext<B> for NetworkContext<'g, 'p, B> {
 
 	/// Send addressed message to a peer.
 	fn send_message(&mut self, who: &PeerId, message: Vec<u8>) {
-		self.protocol.send_consensus(who.clone(), ConsensusMessage {
+		self.protocol.send_consensus(who.clone(), vec![ConsensusMessage {
 			engine_id: self.engine_id,
 			data: message,
-		});
+		}]);
 	}
 
 	/// Send all messages with given topic to a peer.
@@ -190,7 +190,7 @@ fn propagate<'a, B: BlockT, I>(
 	peers: &mut HashMap<PeerId, PeerConsensus<B::Hash>>,
 	validators: &HashMap<ConsensusEngineId, Arc<dyn Validator<B>>>,
 )
-	where I: IntoIterator<Item=(&'a B::Hash, &'a B::Hash, &'a ConsensusMessage)>,  // (msg_hash, topic, message)
+	where I: Clone + IntoIterator<Item=(&'a B::Hash, &'a B::Hash, &'a ConsensusMessage)>,  // (msg_hash, topic, message)
 {
 	let mut check_fns = HashMap::new();
 	let mut message_allowed = move |who: &PeerId, intent: MessageIntent, topic: &B::Hash, message: &ConsensusMessage| {
@@ -206,8 +206,9 @@ fn propagate<'a, B: BlockT, I>(
 		(check_fn)(who, intent, topic, &message.data)
 	};
 
-	for (message_hash, topic, message) in messages {
 	for (id, ref mut peer) in peers.iter_mut() {
+		let mut batch = Vec::new();
+		for (message_hash, topic, message) in messages.clone() {
 			let previous_attempts = peer.filtered_messages
 				.get(&message_hash)
 				.cloned()
@@ -245,8 +246,9 @@ fn propagate<'a, B: BlockT, I>(
 			peer.known_messages.insert(message_hash.clone());
 
 			trace!(target: "gossip", "Propagating to {}: {:?}", id, message);
-			protocol.send_consensus(id.clone(), message.clone());
+			batch.push(message.clone())
 		}
+		protocol.send_consensus(id.clone(), batch);
 	}
 }
 
@@ -477,14 +479,16 @@ impl<B: BlockT> ConsensusGossip<B> {
 		&mut self,
 		protocol: &mut dyn Context<B>,
 		who: PeerId,
-		message: ConsensusMessage,
+		messages: Vec<ConsensusMessage>,
 	) {
+		trace!(target:"gossip", "Received {} messages from peer {}", messages.len(), who);
+		for message in messages {
 			let message_hash = HashFor::<B>::hash(&message.data[..]);
 
 			if self.known_messages.contains_key(&message_hash) {
 				trace!(target:"gossip", "Ignored already known message from {}", who);
 				protocol.report_peer(who.clone(), DUPLICATE_GOSSIP_REPUTATION_CHANGE);
-			return;
+				continue;
 			}
 
 			let engine_id = message.engine_id;
@@ -503,8 +507,8 @@ impl<B: BlockT> ConsensusGossip<B> {
 				None => {
 					trace!(target:"gossip", "Unknown message engine id {:?} from {}", engine_id, who);
 					protocol.report_peer(who.clone(), UNKNOWN_GOSSIP_REPUTATION_CHANGE);
-				protocol.disconnect_peer(who);
+					protocol.disconnect_peer(who.clone());
-				return;
+					continue;
 				}
 			};
 
@@ -538,6 +542,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 				trace!(target:"gossip", "Handled valid one hop message from peer {}", who);
 			}
 		}
+	}
 
 	/// Send all messages with given topic to a peer.
 	pub fn send_topic(
@@ -555,6 +560,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 		};
 
 		if let Some(ref mut peer) = self.peers.get_mut(who) {
+			let mut batch = Vec::new();
 			for entry in self.messages.iter().filter(|m| m.topic == topic && m.message.engine_id == engine_id) {
 				let intent = if force {
 					MessageIntent::ForcedBroadcast
@@ -585,11 +591,12 @@ impl<B: BlockT> ConsensusGossip<B> {
 				peer.known_messages.insert(entry.message_hash.clone());
 
 				trace!(target: "gossip", "Sending topic message to {}: {:?}", who, entry.message);
-				protocol.send_consensus(who.clone(), ConsensusMessage {
+				batch.push(ConsensusMessage {
 					engine_id: engine_id.clone(),
 					data: entry.message.data.clone(),
 				});
 			}
+			protocol.send_consensus(who.clone(), batch);
 		}
 	}
 
@@ -626,8 +633,7 @@ impl<B: BlockT> ConsensusGossip<B> {
 
 		peer.filtered_messages.remove(&message_hash);
 		peer.known_messages.insert(message_hash);
-
+		protocol.send_consensus(who.clone(), vec![message.clone()]);
-		protocol.send_consensus(who.clone(), message.clone());
 	}
 }
 
@@ -812,7 +818,7 @@ mod tests {
 		impl<B: BlockT> Context<B> for DummyNetworkContext {
 			fn report_peer(&mut self, _who: PeerId, _reputation: i32) {}
 			fn disconnect_peer(&mut self, _who: PeerId) {}
-			fn send_consensus(&mut self, _who: PeerId, _consensus: ConsensusMessage) {}
+			fn send_consensus(&mut self, _who: PeerId, _consensus: Vec<ConsensusMessage>) {}
 			fn send_chain_specific(&mut self, _who: PeerId, _message: Vec<u8>) {}
 		}
 

substrate/core/network/src/protocol/message.rs

@@ -217,6 +217,8 @@ pub mod generic {
 		FinalityProofRequest(FinalityProofRequest<Hash>),
 		/// Finality proof reponse.
 		FinalityProofResponse(FinalityProofResponse<Hash>),
+		/// Batch of consensus protocol messages.
+		ConsensusBatch(Vec<ConsensusMessage>),
 		/// Chain-specific message.
 		#[codec(index = "255")]
 		ChainSpecific(Vec<u8>),
@@ -243,6 +245,7 @@ pub mod generic {
 				Message::RemoteReadChildRequest(_) => "RemoteReadChildRequest",
 				Message::FinalityProofRequest(_) => "FinalityProofRequest",
 				Message::FinalityProofResponse(_) => "FinalityProofResponse",
+				Message::ConsensusBatch(_) => "ConsensusBatch",
 				Message::ChainSpecific(_) => "ChainSpecific",
 			}
 		}