Extract consensus_gossip.rs and put it in its own crate (#4284)

* Extract gossiping system from network * Finish porting GRANDPA tests * Try put correct engine ID * Fix messages encoding * Fix communication tests * Use a threads pool to spawn stuff * Fix compilation everywhere * Fix bad merge conflict * Remove dependency on async-std * Apply suggestions from code review Co-Authored-By: Robert Habermeier <rphmeier@gmail.com> * More suggestions * Remove network startup GP future * Update to futures_timer * adjust wait_when_behind test * Pass correct Roles after handshake * Revert "adjust wait_when_behind test" This reverts commit 23cb3a0a6d25ed732c2cd648607bc44ef2ab0919. * Crate root documentation * Remove MessageRecipient * Address concerns * Fix more concerns * Forgot Cargo.lock
2026-04-28 05:07:55 +00:00 · 2019-12-13 19:16:10 +01:00
parent 21cbd80f8c
commit c66c191b68
24 changed files with 1087 additions and 624 deletions
@@ -17,7 +17,7 @@
 use crate::{DiscoveryNetBehaviour, config::ProtocolId};
 use legacy_proto::{LegacyProto, LegacyProtoOut};
 use crate::utils::interval;
-use bytes::BytesMut;
+use bytes::{Bytes, BytesMut};
 use futures::prelude::*;
 use futures03::{StreamExt as _, TryStreamExt as _};
 use libp2p::{Multiaddr, PeerId};
@@ -38,7 +38,6 @@ use sp_runtime::traits::{
 use sp_arithmetic::traits::SaturatedConversion;
 use message::{BlockAnnounce, BlockAttributes, Direction, FromBlock, Message, RequestId};
 use message::generic::{Message as GenericMessage, ConsensusMessage};
-use consensus_gossip::{ConsensusGossip, MessageRecipient as GossipMessageRecipient};
 use light_dispatch::{LightDispatch, LightDispatchNetwork, RequestData};
 use specialization::NetworkSpecialization;
 use sync::{ChainSync, SyncState};
@@ -58,7 +57,6 @@ use util::LruHashSet;
 mod legacy_proto;
 mod util;

-pub mod consensus_gossip;
 pub mod message;
 pub mod event;
 pub mod light_dispatch;
@@ -135,7 +133,6 @@ pub struct Protocol<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> {
 	genesis_hash: B::Hash,
 	sync: ChainSync<B>,
 	specialization: S,
-	consensus_gossip: ConsensusGossip<B>,
 	context_data: ContextData<B, H>,
 	/// List of nodes for which we perform additional logging because they are important for the
 	/// user.
@@ -149,6 +146,8 @@ pub struct Protocol<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> {
 	finality_proof_provider: Option<Arc<dyn FinalityProofProvider<B>>>,
 	/// Handles opening the unique substream and sending and receiving raw messages.
 	behaviour: LegacyProto<Substream<StreamMuxerBox>>,
+	/// List of notification protocols that have been registered.
+	registered_notif_protocols: HashSet<ConsensusEngineId>,
 }

 #[derive(Default)]
@@ -473,13 +472,13 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 			genesis_hash: info.chain.genesis_hash,
 			sync,
 			specialization,
-			consensus_gossip: ConsensusGossip::new(),
 			handshaking_peers: HashMap::new(),
 			important_peers,
 			transaction_pool,
 			finality_proof_provider,
 			peerset_handle: peerset_handle.clone(),
 			behaviour,
+			registered_notif_protocols: HashSet::new(),
 		};

 		Ok((protocol, peerset_handle))
@@ -614,7 +613,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		stats.count_in += 1;

 		match message {
-			GenericMessage::Status(s) => self.on_status_message(who, s),
+			GenericMessage::Status(s) => return self.on_status_message(who, s),
 			GenericMessage::BlockRequest(r) => self.on_block_request(who, r),
 			GenericMessage::BlockResponse(r) => {
 				// Note, this is safe because only `ordinary bodies` and `remote bodies` are received in this matter.
@@ -656,20 +655,38 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 				return self.on_finality_proof_response(who, response),
 			GenericMessage::RemoteReadChildRequest(request) =>
 				self.on_remote_read_child_request(who, request),
-			GenericMessage::Consensus(msg) => {
-				self.consensus_gossip.on_incoming(
-					&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),
-					who,
-					vec![msg],
-				);
-			}
+			GenericMessage::Consensus(msg) =>
+				return if self.registered_notif_protocols.contains(&msg.engine_id) {
+					CustomMessageOutcome::NotificationsReceived {
+						remote: who.clone(),
+						messages: vec![(msg.engine_id, From::from(msg.data))],
+					}
+				} else {
+					warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
+					CustomMessageOutcome::None
+				},
 			GenericMessage::ConsensusBatch(messages) => {
-				self.consensus_gossip.on_incoming(
-					&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),
-					who,
-					messages,
-				);
-			}
+				let messages = messages
+					.into_iter()
+					.filter_map(|msg| {
+						if self.registered_notif_protocols.contains(&msg.engine_id) {
+							Some((msg.engine_id, From::from(msg.data)))
+						} else {
+							warn!(target: "sync", "Received message on non-registered protocol: {:?}", msg.engine_id);
+							None
+						}
+					})
+					.collect::<Vec<_>>();
+
+				return if !messages.is_empty() {
+					CustomMessageOutcome::NotificationsReceived {
+						remote: who.clone(),
+						messages,
+					}
+				} else {
+					CustomMessageOutcome::None
+				};
+			},
 			GenericMessage::ChainSpecific(msg) => self.specialization.on_message(
 				&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle),
 				who,
@@ -699,14 +716,6 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		);
 	}

-	/// Locks `self` and returns a context plus the `ConsensusGossip` struct.
-	pub fn consensus_gossip_lock<'a>(
-		&'a mut self,
-	) -> (impl Context<B> + 'a, &'a mut ConsensusGossip<B>) {
-		let context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle);
-		(context, &mut self.consensus_gossip)
-	}
-
 	/// Locks `self` and returns a context plus the network specialization.
 	pub fn specialization_lock<'a>(
 		&'a mut self,
@@ -715,26 +724,6 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		(context, &mut self.specialization)
 	}

-	/// Gossip a consensus message to the network.
-	pub fn gossip_consensus_message(
-		&mut self,
-		topic: B::Hash,
-		engine_id: ConsensusEngineId,
-		message: Vec<u8>,
-		recipient: GossipMessageRecipient,
-	) {
-		let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle);
-		let message = ConsensusMessage { data: message, engine_id };
-		match recipient {
-			GossipMessageRecipient::BroadcastToAll =>
-				self.consensus_gossip.multicast(&mut context, topic, message, true),
-			GossipMessageRecipient::BroadcastNew =>
-				self.consensus_gossip.multicast(&mut context, topic, message, false),
-			GossipMessageRecipient::Peer(who) =>
-				self.send_message(&who, GenericMessage::Consensus(message)),
-		}
-	}
-
 	/// Called when a new peer is connected
 	pub fn on_peer_connected(&mut self, who: PeerId) {
 		trace!(target: "sync", "Connecting {}", who);
@@ -755,11 +744,8 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 			self.handshaking_peers.remove(&peer);
 			self.context_data.peers.remove(&peer)
 		};
-		if let Some(peer_data) = removed {
+		if let Some(_peer_data) = removed {
 			let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle);
-			if peer_data.info.protocol_version > 2 {
-				self.consensus_gossip.peer_disconnected(&mut context, peer.clone());
-			}
 			self.sync.peer_disconnected(peer.clone());
 			self.specialization.on_disconnect(&mut context, peer.clone());
 			self.light_dispatch.on_disconnect(LightDispatchIn {
@@ -922,9 +908,6 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 	///
 	/// > **Note**: This method normally doesn't have to be called except for testing purposes.
 	pub fn tick(&mut self) {
-		self.consensus_gossip.tick(
-			&mut ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle)
-		);
 		self.maintain_peers();
 		self.light_dispatch.maintain_peers(LightDispatchIn {
 			behaviour: &mut self.behaviour,
@@ -975,9 +958,9 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 	}

 	/// Called by peer to report status
-	fn on_status_message(&mut self, who: PeerId, status: message::Status<B>) {
+	fn on_status_message(&mut self, who: PeerId, status: message::Status<B>) -> CustomMessageOutcome<B> {
 		trace!(target: "sync", "New peer {} {:?}", who, status);
-		let protocol_version = {
+		let _protocol_version = {
 			if self.context_data.peers.contains_key(&who) {
 				log!(
 					target: "sync",
@@ -985,7 +968,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 					"Unexpected status packet from {}", who
 				);
 				self.peerset_handle.report_peer(who, rep::UNEXPECTED_STATUS);
-				return;
+				return CustomMessageOutcome::None;
 			}
 			if status.genesis_hash != self.genesis_hash {
 				log!(
@@ -996,7 +979,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 				);
 				self.peerset_handle.report_peer(who.clone(), rep::GENESIS_MISMATCH);
 				self.behaviour.disconnect_peer(&who);
-				return;
+				return CustomMessageOutcome::None;
 			}
 			if status.version < MIN_VERSION && CURRENT_VERSION < status.min_supported_version {
 				log!(
@@ -1006,7 +989,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 				);
 				self.peerset_handle.report_peer(who.clone(), rep::BAD_PROTOCOL);
 				self.behaviour.disconnect_peer(&who);
-				return;
+				return CustomMessageOutcome::None;
 			}

 			if self.config.roles.is_light() {
@@ -1015,7 +998,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 					debug!(target: "sync", "Peer {} is unable to serve light requests", who);
 					self.peerset_handle.report_peer(who.clone(), rep::BAD_ROLE);
 					self.behaviour.disconnect_peer(&who);
-					return;
+					return CustomMessageOutcome::None;
 				}

 				// we don't interested in peers that are far behind us
@@ -1032,7 +1015,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 					debug!(target: "sync", "Peer {} is far behind us and will unable to serve light requests", who);
 					self.peerset_handle.report_peer(who.clone(), rep::PEER_BEHIND_US_LIGHT);
 					self.behaviour.disconnect_peer(&who);
-					return;
+					return CustomMessageOutcome::None;
 				}
 			}

@@ -1047,7 +1030,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 				},
 				None => {
 					error!(target: "sync", "Received status from previously unconnected node {}", who);
-					return;
+					return CustomMessageOutcome::None;
 				},
 			};

@@ -1082,11 +1065,64 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 				}
 			}
 		}
+
 		let mut context = ProtocolContext::new(&mut self.context_data, &mut self.behaviour, &self.peerset_handle);
-		if protocol_version > 2 {
-			self.consensus_gossip.new_peer(&mut context, who.clone(), status.roles);
+		self.specialization.on_connect(&mut context, who.clone(), status);
+
+		// Notify all the notification protocols as open.
+		CustomMessageOutcome::NotificationStreamOpened {
+			remote: who,
+			protocols: self.registered_notif_protocols.iter().cloned().collect(),
+			roles: info.roles,
 		}
-		self.specialization.on_connect(&mut context, who, status);
+	}
+
+	/// Send a notification to the given peer we're connected to.
+	///
+	/// Doesn't do anything if we don't have a notifications substream for that protocol with that
+	/// peer.
+	pub fn write_notification(
+		&mut self,
+		target: PeerId,
+		engine_id: ConsensusEngineId,
+		message: impl Into<Vec<u8>>
+	) {
+		if !self.registered_notif_protocols.contains(&engine_id) {
+			error!(
+				target: "sub-libp2p",
+				"Sending a notification with a protocol that wasn't registered: {:?}",
+				engine_id
+			);
+		}
+
+		self.send_message(&target, GenericMessage::Consensus(ConsensusMessage {
+			engine_id,
+			data: message.into(),
+		}));
+	}
+
+	/// Registers a new notifications protocol.
+	///
+	/// You are very strongly encouraged to call this method very early on. Any connection open
+	/// will retain the protocols that were registered then, and not any new one.
+	pub fn register_notifications_protocol(
+		&mut self,
+		engine_id: ConsensusEngineId,
+	) -> Vec<event::Event> {
+		if !self.registered_notif_protocols.insert(engine_id) {
+			error!(target: "sub-libp2p", "Notifications protocol already registered: {:?}", engine_id);
+		}
+
+		// Registering a protocol while we already have open connections isn't great, but for now
+		// we handle it by notifying that we opened channels with everyone.
+		self.context_data.peers.iter()
+			.map(|(peer_id, peer)|
+				event::Event::NotificationStreamOpened {
+					remote: peer_id.clone(),
+					engine_id,
+					roles: peer.info.roles,
+				})
+			.collect()
 	}

 	/// Called when peer sends us new extrinsics
@@ -1758,6 +1794,12 @@ pub enum CustomMessageOutcome<B: BlockT> {
 	BlockImport(BlockOrigin, Vec<IncomingBlock<B>>),
 	JustificationImport(Origin, B::Hash, NumberFor<B>, Justification),
 	FinalityProofImport(Origin, B::Hash, NumberFor<B>, Vec<u8>),
+	/// Notification protocols have been opened with a remote.
+	NotificationStreamOpened { remote: PeerId, protocols: Vec<ConsensusEngineId>, roles: Roles },
+	/// Notification protocols have been closed with a remote.
+	NotificationsStreamClosed { remote: PeerId, protocols: Vec<ConsensusEngineId> },
+	/// Messages have been received on one or more notifications protocols.
+	NotificationsReceived { remote: PeerId, messages: Vec<(ConsensusEngineId, Bytes)> },
 	None,
 }

@@ -1887,12 +1929,16 @@ Protocol<B, S, H> {
 					version <= CURRENT_VERSION as u8
 					&& version >= MIN_VERSION as u8
 				);
-				self.on_peer_connected(peer_id);
+				self.on_peer_connected(peer_id.clone());
 				CustomMessageOutcome::None
 			}
 			LegacyProtoOut::CustomProtocolClosed { peer_id, .. } => {
-				self.on_peer_disconnected(peer_id);
-				CustomMessageOutcome::None
+				self.on_peer_disconnected(peer_id.clone());
+				// Notify all the notification protocols as closed.
+				CustomMessageOutcome::NotificationsStreamClosed {
+					remote: peer_id,
+					protocols: self.registered_notif_protocols.iter().cloned().collect(),
+				}
 			},
 			LegacyProtoOut::CustomMessage { peer_id, message } =>
 				self.on_custom_message(peer_id, message),