Upgrade to libp2p 0.51.3 (#13587)

* client/network: upgrade to libp2p 0.51.0 * make discovery.rs compile * make peer_info.rs compile * changes to notifications and request-response proto * make service.rs compile * towards making request_responses.rs compile * make request_responses.rs compile * make request_responses.rs compile * fix notifications/behaviour.rs tests * fix warnings * remove old code * allow deprecated code (temporary) * upgrade to libp2p 0.51.1 * add TODO for behaviour tests * return empty vec if peer_id is absent https://github.com/paritytech/substrate/pull/13587#discussion_r1141695167 fyi: I don't really know what the old behaviour was. * update comment to reflect new defaults Closes #13338 * Revert "update comment to reflect new defaults" This reverts commit 7a981abd69308e9d522ec94905f181439a1b1dba. * remove config.rs (from wrong merge) * upgrade to libp2p 0.51.2 * fix formatting * use handle_pending_outbound_connection in networt_state RPC * update deps * use re-exports when we use other libp2p packages * Apply suggestions from code review Co-authored-by: Dmitry Markin <dmitry@markin.tech> * format code * handle potential errors in network_state RPC * only update libp2p crate * update libp2p-core * fix docs * use libp2p-identity instead of libp2p where it's possible. libp2p-identity is much smaller, hence makes sense to use it instead of larger libp2p crate. * Update client/network/src/discovery.rs Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> * update Cargo.lock * add comment for per_connection_event_buffer_size current value is somewhat arbitrary and needs to be tweaked depending on memory usage and network worker sleep stats. * fix link format * update Cargo.lock * upgrade to libp2p 0.51.3 * deprecate mplex * Revert "deprecate mplex" This reverts commit 9e25820e706e464a0e962a8604861fcb2a7641eb. * Revert "upgrade to libp2p 0.51.3" This reverts commit 6544dd4138e2f89517bd7c7281fc78a638ec7040. * use new libp2p version in `statement` crate * pin version temporarily * libp2p 0.51.3 * deprecate mplex * deprecate legacy noise handshake * fix build error * update libp2p-identity * enable libp2p-identity:ed25519 feature in sc-consensus * enable ed25519 for peerset as well --------- Co-authored-by: Dmitry Markin <dmitry@markin.tech> Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> Co-authored-by: parity-processbot <>
2026-06-13 17:31:05 +00:00 · 2023-05-12 11:12:51 +04:00
parent 56940bc874
commit e4b1aa1811
48 changed files with 1005 additions and 927 deletions
@@ -53,26 +53,24 @@ use futures::prelude::*;
 use futures_timer::Delay;
 use ip_network::IpNetwork;
 use libp2p::{
-	core::{connection::ConnectionId, Multiaddr, PeerId, PublicKey},
+	core::{Endpoint, Multiaddr},
 	kad::{
-		handler::KademliaHandlerProto,
-		record::{
-			self,
-			store::{MemoryStore, RecordStore},
-		},
+		handler::KademliaHandler,
+		record::store::{MemoryStore, RecordStore},
 		GetClosestPeersError, GetRecordOk, Kademlia, KademliaBucketInserts, KademliaConfig,
-		KademliaEvent, QueryId, QueryResult, Quorum, Record,
+		KademliaEvent, QueryId, QueryResult, Quorum, Record, RecordKey,
 	},
 	mdns::{self, tokio::Behaviour as TokioMdns},
 	multiaddr::Protocol,
 	swarm::{
 		behaviour::{
-			toggle::{Toggle, ToggleIntoConnectionHandler},
+			toggle::{Toggle, ToggleConnectionHandler},
 			DialFailure, FromSwarm, NewExternalAddr,
 		},
-		ConnectionHandler, DialError, IntoConnectionHandler, NetworkBehaviour,
-		NetworkBehaviourAction, PollParameters,
+		ConnectionDenied, ConnectionId, DialError, NetworkBehaviour, PollParameters, THandler,
+		THandlerInEvent, THandlerOutEvent, ToSwarm,
 	},
+	PeerId,
 };
 use log::{debug, info, trace, warn};
 use sp_core::hexdisplay::HexDisplay;
@@ -107,9 +105,9 @@ pub struct DiscoveryConfig {

 impl DiscoveryConfig {
 	/// Create a default configuration with the given public key.
-	pub fn new(local_public_key: PublicKey) -> Self {
+	pub fn new(local_peer_id: PeerId) -> Self {
 		Self {
-			local_peer_id: local_public_key.to_peer_id(),
+			local_peer_id,
 			permanent_addresses: Vec::new(),
 			dht_random_walk: true,
 			allow_private_ip: true,
@@ -235,7 +233,7 @@ impl DiscoveryConfig {
 			allow_private_ip,
 			discovery_only_if_under_num,
 			mdns: if enable_mdns {
-				match TokioMdns::new(mdns::Config::default()) {
+				match TokioMdns::new(mdns::Config::default(), local_peer_id) {
 					Ok(mdns) => Some(mdns),
 					Err(err) => {
 						warn!(target: "sub-libp2p", "Failed to initialize mDNS: {:?}", err);
@@ -374,7 +372,7 @@ impl DiscoveryBehaviour {
 	/// Start fetching a record from the DHT.
 	///
 	/// A corresponding `ValueFound` or `ValueNotFound` event will later be generated.
-	pub fn get_value(&mut self, key: record::Key) {
+	pub fn get_value(&mut self, key: RecordKey) {
 		if let Some(k) = self.kademlia.as_mut() {
 			k.get_record(key.clone());
 		}
@@ -384,7 +382,7 @@ impl DiscoveryBehaviour {
 	/// `get_value`.
 	///
 	/// A corresponding `ValuePut` or `ValuePutFailed` event will later be generated.
-	pub fn put_value(&mut self, key: record::Key, value: Vec<u8>) {
+	pub fn put_value(&mut self, key: RecordKey, value: Vec<u8>) {
 		if let Some(k) = self.kademlia.as_mut() {
 			if let Err(e) = k.put_record(Record::new(key.clone(), value.clone()), Quorum::All) {
 				warn!(target: "sub-libp2p", "Libp2p => Failed to put record: {:?}", e);
@@ -460,22 +458,22 @@ pub enum DiscoveryOut {
 	/// The DHT yielded results for the record request.
 	///
 	/// Returning the result grouped in (key, value) pairs as well as the request duration.
-	ValueFound(Vec<(record::Key, Vec<u8>)>, Duration),
+	ValueFound(Vec<(RecordKey, Vec<u8>)>, Duration),

 	/// The record requested was not found in the DHT.
 	///
 	/// Returning the corresponding key as well as the request duration.
-	ValueNotFound(record::Key, Duration),
+	ValueNotFound(RecordKey, Duration),

 	/// The record with a given key was successfully inserted into the DHT.
 	///
 	/// Returning the corresponding key as well as the request duration.
-	ValuePut(record::Key, Duration),
+	ValuePut(RecordKey, Duration),

 	/// Inserting a value into the DHT failed.
 	///
 	/// Returning the corresponding key as well as the request duration.
-	ValuePutFailed(record::Key, Duration),
+	ValuePutFailed(RecordKey, Duration),

 	/// Started a random Kademlia query.
 	///
@@ -484,29 +482,83 @@ pub enum DiscoveryOut {
 }

 impl NetworkBehaviour for DiscoveryBehaviour {
-	type ConnectionHandler = ToggleIntoConnectionHandler<KademliaHandlerProto<QueryId>>;
+	type ConnectionHandler = ToggleConnectionHandler<KademliaHandler<QueryId>>;
 	type OutEvent = DiscoveryOut;

-	fn new_handler(&mut self) -> Self::ConnectionHandler {
-		self.kademlia.new_handler()
+	fn handle_established_inbound_connection(
+		&mut self,
+		connection_id: ConnectionId,
+		peer: PeerId,
+		local_addr: &Multiaddr,
+		remote_addr: &Multiaddr,
+	) -> Result<THandler<Self>, ConnectionDenied> {
+		self.kademlia.handle_established_inbound_connection(
+			connection_id,
+			peer,
+			local_addr,
+			remote_addr,
+		)
 	}

-	fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
+	fn handle_established_outbound_connection(
+		&mut self,
+		connection_id: ConnectionId,
+		peer: PeerId,
+		addr: &Multiaddr,
+		role_override: Endpoint,
+	) -> Result<THandler<Self>, ConnectionDenied> {
+		self.kademlia.handle_established_outbound_connection(
+			connection_id,
+			peer,
+			addr,
+			role_override,
+		)
+	}
+
+	fn handle_pending_inbound_connection(
+		&mut self,
+		connection_id: ConnectionId,
+		local_addr: &Multiaddr,
+		remote_addr: &Multiaddr,
+	) -> Result<(), ConnectionDenied> {
+		self.kademlia
+			.handle_pending_inbound_connection(connection_id, local_addr, remote_addr)
+	}
+
+	fn handle_pending_outbound_connection(
+		&mut self,
+		connection_id: ConnectionId,
+		maybe_peer: Option<PeerId>,
+		addresses: &[Multiaddr],
+		effective_role: Endpoint,
+	) -> Result<Vec<Multiaddr>, ConnectionDenied> {
+		let Some(peer_id) = maybe_peer else { return Ok(Vec::new()); };
+
 		let mut list = self
 			.permanent_addresses
 			.iter()
-			.filter_map(|(p, a)| if p == peer_id { Some(a.clone()) } else { None })
+			.filter_map(|(p, a)| (*p == peer_id).then_some(a.clone()))
 			.collect::<Vec<_>>();

-		if let Some(ephemeral_addresses) = self.ephemeral_addresses.get(peer_id) {
+		if let Some(ephemeral_addresses) = self.ephemeral_addresses.get(&peer_id) {
 			list.extend(ephemeral_addresses.clone());
 		}

 		{
-			let mut list_to_filter = self.kademlia.addresses_of_peer(peer_id);
+			let mut list_to_filter = self.kademlia.handle_pending_outbound_connection(
+				connection_id,
+				maybe_peer,
+				addresses,
+				effective_role,
+			)?;

 			if let Some(ref mut mdns) = self.mdns {
-				list_to_filter.extend(mdns.addresses_of_peer(peer_id));
+				list_to_filter.extend(mdns.handle_pending_outbound_connection(
+					connection_id,
+					maybe_peer,
+					addresses,
+					effective_role,
+				)?);
 			}

 			if !self.allow_private_ip {
@@ -522,7 +574,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {

 		trace!(target: "sub-libp2p", "Addresses of {:?}: {:?}", peer_id, list);

-		list
+		Ok(list)
 	}

 	fn on_swarm_event(&mut self, event: FromSwarm<Self::ConnectionHandler>) {
@@ -603,8 +655,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 		&mut self,
 		peer_id: PeerId,
 		connection_id: ConnectionId,
-		event: <<Self::ConnectionHandler as IntoConnectionHandler>::Handler as
-      ConnectionHandler>::OutEvent,
+		event: THandlerOutEvent<Self>,
 	) {
 		self.kademlia.on_connection_handler_event(peer_id, connection_id, event);
 	}
@@ -613,10 +664,10 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 		&mut self,
 		cx: &mut Context,
 		params: &mut impl PollParameters,
-	) -> Poll<NetworkBehaviourAction<Self::OutEvent, Self::ConnectionHandler>> {
+	) -> Poll<ToSwarm<Self::OutEvent, THandlerInEvent<Self>>> {
 		// Immediately process the content of `discovered`.
 		if let Some(ev) = self.pending_events.pop_front() {
-			return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+			return Poll::Ready(ToSwarm::GenerateEvent(ev))
 		}

 		// Poll the stream that fires when we need to start a random Kademlia query.
@@ -650,7 +701,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {

 					if actually_started {
 						let ev = DiscoveryOut::RandomKademliaStarted;
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					}
 				}
 			}
@@ -658,18 +709,18 @@ impl NetworkBehaviour for DiscoveryBehaviour {

 		while let Poll::Ready(ev) = self.kademlia.poll(cx, params) {
 			match ev {
-				NetworkBehaviourAction::GenerateEvent(ev) => match ev {
+				ToSwarm::GenerateEvent(ev) => match ev {
 					KademliaEvent::RoutingUpdated { peer, .. } => {
 						let ev = DiscoveryOut::Discovered(peer);
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					},
 					KademliaEvent::UnroutablePeer { peer, .. } => {
 						let ev = DiscoveryOut::UnroutablePeer(peer);
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					},
 					KademliaEvent::RoutablePeer { peer, .. } => {
 						let ev = DiscoveryOut::Discovered(peer);
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					},
 					KademliaEvent::PendingRoutablePeer { .. } |
 					KademliaEvent::InboundRequest { .. } => {
@@ -777,7 +828,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 								)
 							},
 						};
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					},
 					KademliaEvent::OutboundQueryProgressed {
 						result: QueryResult::PutRecord(res),
@@ -799,7 +850,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 								)
 							},
 						};
-						return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+						return Poll::Ready(ToSwarm::GenerateEvent(ev))
 					},
 					KademliaEvent::OutboundQueryProgressed {
 						result: QueryResult::RepublishRecord(res),
@@ -821,24 +872,13 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 						warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e)
 					},
 				},
-				NetworkBehaviourAction::Dial { opts, handler } =>
-					return Poll::Ready(NetworkBehaviourAction::Dial { opts, handler }),
-				NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } =>
-					return Poll::Ready(NetworkBehaviourAction::NotifyHandler {
-						peer_id,
-						handler,
-						event,
-					}),
-				NetworkBehaviourAction::ReportObservedAddr { address, score } =>
-					return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
-						address,
-						score,
-					}),
-				NetworkBehaviourAction::CloseConnection { peer_id, connection } =>
-					return Poll::Ready(NetworkBehaviourAction::CloseConnection {
-						peer_id,
-						connection,
-					}),
+				ToSwarm::Dial { opts } => return Poll::Ready(ToSwarm::Dial { opts }),
+				ToSwarm::NotifyHandler { peer_id, handler, event } =>
+					return Poll::Ready(ToSwarm::NotifyHandler { peer_id, handler, event }),
+				ToSwarm::ReportObservedAddr { address, score } =>
+					return Poll::Ready(ToSwarm::ReportObservedAddr { address, score }),
+				ToSwarm::CloseConnection { peer_id, connection } =>
+					return Poll::Ready(ToSwarm::CloseConnection { peer_id, connection }),
 			}
 		}

@@ -846,7 +886,7 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 		if let Some(ref mut mdns) = self.mdns {
 			while let Poll::Ready(ev) = mdns.poll(cx, params) {
 				match ev {
-					NetworkBehaviourAction::GenerateEvent(event) => match event {
+					ToSwarm::GenerateEvent(event) => match event {
 						mdns::Event::Discovered(list) => {
 							if self.num_connections >= self.discovery_only_if_under_num {
 								continue
@@ -855,25 +895,21 @@ impl NetworkBehaviour for DiscoveryBehaviour {
 							self.pending_events
 								.extend(list.map(|(peer_id, _)| DiscoveryOut::Discovered(peer_id)));
 							if let Some(ev) = self.pending_events.pop_front() {
-								return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev))
+								return Poll::Ready(ToSwarm::GenerateEvent(ev))
 							}
 						},
 						mdns::Event::Expired(_) => {},
 					},
-					NetworkBehaviourAction::Dial { .. } => {
+					ToSwarm::Dial { .. } => {
 						unreachable!("mDNS never dials!");
 					},
-					NetworkBehaviourAction::NotifyHandler { event, .. } => match event {}, /* `event` is an enum with no variant */
-					NetworkBehaviourAction::ReportObservedAddr { address, score } =>
-						return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr {
-							address,
-							score,
-						}),
-					NetworkBehaviourAction::CloseConnection { peer_id, connection } =>
-						return Poll::Ready(NetworkBehaviourAction::CloseConnection {
-							peer_id,
-							connection,
-						}),
+					ToSwarm::NotifyHandler { event, .. } => match event {}, /* `event` is an */
+					// enum with no
+					// variant
+					ToSwarm::ReportObservedAddr { address, score } =>
+						return Poll::Ready(ToSwarm::ReportObservedAddr { address, score }),
+					ToSwarm::CloseConnection { peer_id, connection } =>
+						return Poll::Ready(ToSwarm::CloseConnection { peer_id, connection }),
 				}
 			}
 		}
@@ -912,9 +948,9 @@ mod tests {
 			transport::{MemoryTransport, Transport},
 			upgrade,
 		},
-		identity::{ed25519, Keypair},
+		identity::Keypair,
 		noise,
-		swarm::{Executor, Swarm, SwarmEvent},
+		swarm::{Executor, Swarm, SwarmBuilder, SwarmEvent},
 		yamux, Multiaddr,
 	};
 	use sp_core::hash::H256;
@@ -941,17 +977,14 @@ mod tests {
 			.map(|i| {
 				let keypair = Keypair::generate_ed25519();

-				let noise_keys =
-					noise::Keypair::<noise::X25519Spec>::new().into_authentic(&keypair).unwrap();
-
 				let transport = MemoryTransport::new()
 					.upgrade(upgrade::Version::V1)
-					.authenticate(noise::NoiseConfig::xx(noise_keys).into_authenticated())
-					.multiplex(yamux::YamuxConfig::default())
+					.authenticate(noise::Config::new(&keypair).unwrap())
+					.multiplex(yamux::Config::default())
 					.boxed();

 				let behaviour = {
-					let mut config = DiscoveryConfig::new(keypair.public());
+					let mut config = DiscoveryConfig::new(keypair.public().to_peer_id());
 					config
 						.with_permanent_addresses(first_swarm_peer_id_and_addr.clone())
 						.allow_private_ip(true)
@@ -963,12 +996,13 @@ mod tests {
 				};

 				let runtime = tokio::runtime::Runtime::new().unwrap();
-				let mut swarm = Swarm::with_executor(
+				let mut swarm = SwarmBuilder::with_executor(
 					transport,
 					behaviour,
 					keypair.public().to_peer_id(),
 					TokioExecutor(runtime),
-				);
+				)
+				.build();
 				let listen_addr: Multiaddr =
 					format!("/memory/{}", rand::random::<u64>()).parse().unwrap();

@@ -1070,7 +1104,7 @@ mod tests {

 		let mut discovery = {
 			let keypair = Keypair::generate_ed25519();
-			let mut config = DiscoveryConfig::new(keypair.public());
+			let mut config = DiscoveryConfig::new(keypair.public().to_peer_id());
 			config
 				.allow_private_ip(true)
 				.allow_non_globals_in_dht(true)
@@ -1080,11 +1114,7 @@ mod tests {
 		};

 		let predictable_peer_id = |bytes: &[u8; 32]| {
-			Keypair::Ed25519(ed25519::Keypair::from(
-				ed25519::SecretKey::from_bytes(bytes.to_owned()).unwrap(),
-			))
-			.public()
-			.to_peer_id()
+			Keypair::ed25519_from_bytes(bytes.to_owned()).unwrap().public().to_peer_id()
 		};

 		let remote_peer_id = predictable_peer_id(b"00000000000000000000000000000001");