Integrate litep2p into Polkadot SDK (#2944)

[litep2p](https://github.com/altonen/litep2p) is a libp2p-compatible P2P
networking library. It supports all of the features of `rust-libp2p`
that are currently being utilized by Polkadot SDK.

Compared to `rust-libp2p`, `litep2p` has a quite different architecture
which is why the new `litep2p` network backend is only able to use a
little of the existing code in `sc-network`. The design has been mainly
influenced by how we'd wish to structure our networking-related code in
Polkadot SDK: independent higher-levels protocols directly communicating
with the network over links that support bidirectional backpressure. A
good example would be `NotificationHandle`/`RequestResponseHandle`
abstractions which allow, e.g., `SyncingEngine` to directly communicate
with peers to announce/request blocks.

I've tried running `polkadot --network-backend litep2p` with a few
different peer configurations and there is a noticeable reduction in
networking CPU usage. For high load (`--out-peers 200`), networking CPU
usage goes down from ~110% to ~30% (80 pp) and for normal load
(`--out-peers 40`), the usage goes down from ~55% to ~18% (37 pp).

These should not be taken as final numbers because:

a) there are still some low-hanging optimization fruits, such as
enabling [receive window
auto-tuning](https://github.com/libp2p/rust-yamux/pull/176), integrating
`Peerset` more closely with `litep2p` or improving memory usage of the
WebSocket transport
b) fixing bugs/instabilities that incorrectly cause `litep2p` to do less
work will increase the networking CPU usage
c) verification in a more diverse set of tests/conditions is needed

Nevertheless, these numbers should give an early estimate for CPU usage
of the new networking backend.

This PR consists of three separate changes:
* introduce a generic `PeerId` (wrapper around `Multihash`) so that we
don't have use `NetworkService::PeerId` in every part of the code that
uses a `PeerId`
* introduce `NetworkBackend` trait, implement it for the libp2p network
stack and make Polkadot SDK generic over `NetworkBackend`
  * implement `NetworkBackend` for litep2p

The new library should be considered experimental which is why
`rust-libp2p` will remain as the default option for the time being. This
PR currently depends on the master branch of `litep2p` but I'll cut a
new release for the library once all review comments have been
addresses.

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Alexandru Vasile <alexandru.vasile@parity.io>

substrate/client/network/test/src/block_import.rs

@@ -58,7 +58,7 @@ fn prepare_good_block() -> (TestClient, Hash, u64, PeerId, IncomingBlock<Block>)
 			body: Some(Vec::new()),
 			indexed_body: None,
 			justifications,
-			origin: Some(peer_id),
+			origin: Some(peer_id.into()),
 			allow_missing_state: false,
 			import_existing: false,
 			state: None,
@@ -81,7 +81,7 @@ fn import_single_good_block_works() {
 		&mut PassThroughVerifier::new(true),
 	)) {
 		Ok(BlockImportStatus::ImportedUnknown(ref num, ref aux, ref org))
-			if *num == number && *aux == expected_aux && *org == Some(peer_id) => {},
+			if *num == number && *aux == expected_aux && *org == Some(peer_id.into()) => {},
 		r @ _ => panic!("{:?}", r),
 	}
 }
@@ -110,7 +110,7 @@ fn import_single_good_block_without_header_fails() {
 		block,
 		&mut PassThroughVerifier::new(true),
 	)) {
-		Err(BlockImportError::IncompleteHeader(ref org)) if *org == Some(peer_id) => {},
+		Err(BlockImportError::IncompleteHeader(ref org)) if *org == Some(peer_id.into()) => {},
 		_ => panic!(),
 	}
 }

substrate/client/network/test/src/fuzz.rs

@@ -20,7 +20,6 @@
 //! and `PeerStore` to discover possible inconsistencies in peer management.
 
 use futures::prelude::*;
-use libp2p::PeerId;
 use rand::{
 	distributions::{Distribution, Uniform, WeightedIndex},
 	seq::IteratorRandom,
@@ -28,10 +27,13 @@ use rand::{
 use sc_network::{
 	peer_store::{PeerStore, PeerStoreProvider},
 	protocol_controller::{IncomingIndex, Message, ProtoSetConfig, ProtocolController, SetId},
-	ReputationChange,
+	PeerId, ReputationChange,
 };
 use sc_utils::mpsc::tracing_unbounded;
-use std::collections::{HashMap, HashSet};
+use std::{
+	collections::{HashMap, HashSet},
+	sync::Arc,
+};
 
 /// Peer events as observed by `Notifications` / fuzz test.
 #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
@@ -141,7 +143,7 @@ async fn test_once() {
 		.collect();
 
 	let peer_store = PeerStore::new(bootnodes);
-	let mut peer_store_handle = peer_store.handle();
+	let peer_store_handle = peer_store.handle();
 
 	let (to_notifications, mut from_controller) =
 		tracing_unbounded("test_to_notifications", 10_000);
@@ -163,7 +165,7 @@ async fn test_once() {
 			reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0,
 		},
 		to_notifications,
-		Box::new(peer_store_handle.clone()),
+		Arc::new(peer_store_handle.clone()),
 	);
 
 	tokio::spawn(peer_store.run());
@@ -319,14 +321,15 @@ async fn test_once() {
 				1 => {
 					let new_id = PeerId::random();
 					known_nodes.insert(new_id, State::Disconnected);
-					peer_store_handle.add_known_peer(new_id);
+					peer_store_handle.add_known_peer(new_id.into());
 				},
 
 				// If we generate 2, adjust a random reputation.
 				2 =>
 					if let Some(id) = known_nodes.keys().choose(&mut rng) {
 						let val = Uniform::new_inclusive(i32::MIN, i32::MAX).sample(&mut rng);
-						peer_store_handle.report_peer(*id, ReputationChange::new(val, ""));
+						let peer: sc_network_types::PeerId = id.into();
+						peer_store_handle.report_peer(peer, ReputationChange::new(val, ""));
 					},
 
 				// If we generate 3, disconnect from a random node.
@@ -414,5 +417,6 @@ async fn test_once() {
 			}
 		}
 	})
-	.await;
+	.await
+	.unwrap();
 }

substrate/client/network/test/src/lib.rs

@@ -58,7 +58,7 @@ use sc_network::{
 	request_responses::ProtocolConfig as RequestResponseConfig,
 	types::ProtocolName,
 	Multiaddr, NetworkBlock, NetworkService, NetworkStateInfo, NetworkSyncForkRequest,
-	NetworkWorker, NotificationService,
+	NetworkWorker, NotificationMetrics, NotificationService,
 };
 use sc_network_common::role::Roles;
 use sc_network_light::light_client_requests::handler::LightClientRequestHandler;
@@ -248,7 +248,7 @@ where
 {
 	/// Get this peer ID.
 	pub fn id(&self) -> PeerId {
-		self.network.service().local_peer_id()
+		self.network.service().local_peer_id().into()
 	}
 
 	/// Returns true if we're major syncing.
@@ -295,7 +295,11 @@ where
 		hash: <Block as BlockT>::Hash,
 		number: NumberFor<Block>,
 	) {
-		self.sync_service.set_sync_fork_request(peers, hash, number);
+		self.sync_service.set_sync_fork_request(
+			peers.into_iter().map(From::from).collect(),
+			hash,
+			number,
+		);
 	}
 
 	/// Add blocks to the peer -- edit the block before adding
@@ -829,7 +833,7 @@ pub trait TestNetFactory: Default + Sized + Send {
 
 		let (chain_sync_network_provider, chain_sync_network_handle) =
 			NetworkServiceProvider::new();
-		let mut block_relay_params = BlockRequestHandler::new(
+		let mut block_relay_params = BlockRequestHandler::new::<NetworkWorker<_, _>>(
 			chain_sync_network_handle.clone(),
 			&protocol_id,
 			None,
@@ -841,18 +845,24 @@ pub trait TestNetFactory: Default + Sized + Send {
 		}));
 
 		let state_request_protocol_config = {
-			let (handler, protocol_config) =
-				StateRequestHandler::new(&protocol_id, None, client.clone(), 50);
+			let (handler, protocol_config) = StateRequestHandler::new::<NetworkWorker<_, _>>(
+				&protocol_id,
+				None,
+				client.clone(),
+				50,
+			);
 			self.spawn_task(handler.run().boxed());
 			protocol_config
 		};
 
-		let light_client_request_protocol_config = {
-			let (handler, protocol_config) =
-				LightClientRequestHandler::new(&protocol_id, None, client.clone());
-			self.spawn_task(handler.run().boxed());
-			protocol_config
-		};
+		let light_client_request_protocol_config =
+			{
+				let (handler, protocol_config) = LightClientRequestHandler::new::<
+					NetworkWorker<_, _>,
+				>(&protocol_id, None, client.clone());
+				self.spawn_task(handler.run().boxed());
+				protocol_config
+			};
 
 		let warp_sync = Arc::new(TestWarpSyncProvider(client.clone()));
 
@@ -866,35 +876,45 @@ pub trait TestNetFactory: Default + Sized + Send {
 		};
 
 		let warp_protocol_config = {
-			let (handler, protocol_config) = warp_request_handler::RequestHandler::new(
-				protocol_id.clone(),
-				client
-					.block_hash(0u32.into())
-					.ok()
-					.flatten()
-					.expect("Genesis block exists; qed"),
-				None,
-				warp_sync.clone(),
-			);
+			let (handler, protocol_config) =
+				warp_request_handler::RequestHandler::new::<_, NetworkWorker<_, _>>(
+					protocol_id.clone(),
+					client
+						.block_hash(0u32.into())
+						.ok()
+						.flatten()
+						.expect("Genesis block exists; qed"),
+					None,
+					warp_sync.clone(),
+				);
 			self.spawn_task(handler.run().boxed());
 			protocol_config
 		};
 
 		let peer_store = PeerStore::new(
-			network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(),
+			network_config
+				.boot_nodes
+				.iter()
+				.map(|bootnode| bootnode.peer_id.into())
+				.collect(),
 		);
-		let peer_store_handle = peer_store.handle();
+		let peer_store_handle = Arc::new(peer_store.handle());
 		self.spawn_task(peer_store.run().boxed());
 
 		let block_announce_validator = config
 			.block_announce_validator
 			.unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator));
+		let metrics = <NetworkWorker<_, _> as sc_network::NetworkBackend<
+			Block,
+			<Block as BlockT>::Hash,
+		>>::register_notification_metrics(None);
 
 		let (engine, sync_service, block_announce_config) =
 			sc_network_sync::engine::SyncingEngine::new(
 				Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }),
 				client.clone(),
 				None,
+				metrics,
 				&full_net_config,
 				protocol_id.clone(),
 				&fork_id,
@@ -935,12 +955,13 @@ pub trait TestNetFactory: Default + Sized + Send {
 				tokio::spawn(f);
 			}),
 			network_config: full_net_config,
-			peer_store: peer_store_handle,
 			genesis_hash,
 			protocol_id,
 			fork_id,
 			metrics_registry: None,
 			block_announce_config,
+			bitswap_config: None,
+			notification_metrics: NotificationMetrics::new(None),
 		})
 		.unwrap();
 
@@ -961,8 +982,10 @@ pub trait TestNetFactory: Default + Sized + Send {
 
 		self.mut_peers(move |peers| {
 			for peer in peers.iter_mut() {
-				peer.network
-					.add_known_address(network.service().local_peer_id(), listen_addr.clone());
+				peer.network.add_known_address(
+					network.service().local_peer_id().into(),
+					listen_addr.clone(),
+				);
 			}
 
 			let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse());

substrate/client/network/test/src/service.rs

@@ -17,16 +17,15 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use futures::prelude::*;
-use libp2p::{Multiaddr, PeerId};
 
 use sc_consensus::{ImportQueue, Link};
 use sc_network::{
 	config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig},
 	event::Event,
-	peer_store::PeerStore,
+	peer_store::{PeerStore, PeerStoreProvider},
 	service::traits::{NotificationEvent, ValidationResult},
-	NetworkEventStream, NetworkPeers, NetworkService, NetworkStateInfo, NetworkWorker,
-	NotificationService,
+	Multiaddr, NetworkEventStream, NetworkPeers, NetworkService, NetworkStateInfo, NetworkWorker,
+	NotificationMetrics, NotificationService, PeerId,
 };
 use sc_network_common::role::Roles;
 use sc_network_light::light_client_requests::handler::LightClientRequestHandler;
@@ -159,41 +158,54 @@ impl TestNetworkBuilder {
 
 		let (chain_sync_network_provider, chain_sync_network_handle) =
 			self.chain_sync_network.unwrap_or(NetworkServiceProvider::new());
-		let mut block_relay_params = BlockRequestHandler::new(
-			chain_sync_network_handle.clone(),
-			&protocol_id,
-			None,
-			client.clone(),
-			50,
-		);
+		let mut block_relay_params =
+			BlockRequestHandler::new::<
+				NetworkWorker<
+					substrate_test_runtime_client::runtime::Block,
+					substrate_test_runtime_client::runtime::Hash,
+				>,
+			>(chain_sync_network_handle.clone(), &protocol_id, None, client.clone(), 50);
 		tokio::spawn(Box::pin(async move {
 			block_relay_params.server.run().await;
 		}));
 
 		let state_request_protocol_config = {
-			let (handler, protocol_config) =
-				StateRequestHandler::new(&protocol_id, None, client.clone(), 50);
+			let (handler, protocol_config) = StateRequestHandler::new::<
+				NetworkWorker<
+					substrate_test_runtime_client::runtime::Block,
+					substrate_test_runtime_client::runtime::Hash,
+				>,
+			>(&protocol_id, None, client.clone(), 50);
 			tokio::spawn(handler.run().boxed());
 			protocol_config
 		};
 
 		let light_client_request_protocol_config = {
-			let (handler, protocol_config) =
-				LightClientRequestHandler::new(&protocol_id, None, client.clone());
+			let (handler, protocol_config) = LightClientRequestHandler::new::<
+				NetworkWorker<
+					substrate_test_runtime_client::runtime::Block,
+					substrate_test_runtime_client::runtime::Hash,
+				>,
+			>(&protocol_id, None, client.clone());
 			tokio::spawn(handler.run().boxed());
 			protocol_config
 		};
 
 		let peer_store = PeerStore::new(
-			network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(),
+			network_config
+				.boot_nodes
+				.iter()
+				.map(|bootnode| bootnode.peer_id.into())
+				.collect(),
 		);
-		let peer_store_handle = peer_store.handle();
+		let peer_store_handle: Arc<dyn PeerStoreProvider> = Arc::new(peer_store.handle());
 		tokio::spawn(peer_store.run().boxed());
 
 		let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new(
 			Roles::from(&config::Role::Full),
 			client.clone(),
 			None,
+			NotificationMetrics::new(None),
 			&full_net_config,
 			protocol_id.clone(),
 			&None,
@@ -204,7 +216,7 @@ impl TestNetworkBuilder {
 			block_relay_params.downloader,
 			state_request_protocol_config.name.clone(),
 			None,
-			peer_store_handle.clone(),
+			Arc::clone(&peer_store_handle),
 		)
 		.unwrap();
 		let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone()));
@@ -239,7 +251,11 @@ impl TestNetworkBuilder {
 		let worker = NetworkWorker::<
 			substrate_test_runtime_client::runtime::Block,
 			substrate_test_runtime_client::runtime::Hash,
-		>::new(config::Params::<substrate_test_runtime_client::runtime::Block> {
+		>::new(config::Params::<
+			substrate_test_runtime_client::runtime::Block,
+			substrate_test_runtime_client::runtime::Hash,
+			NetworkWorker<_, _>,
+		> {
 			block_announce_config,
 			role: config::Role::Full,
 			executor: Box::new(|f| {
@@ -247,10 +263,11 @@ impl TestNetworkBuilder {
 			}),
 			genesis_hash,
 			network_config: full_net_config,
-			peer_store: peer_store_handle,
 			protocol_id,
 			fork_id,
 			metrics_registry: None,
+			bitswap_config: None,
+			notification_metrics: NotificationMetrics::new(None),
 		})
 		.unwrap();
 
@@ -670,7 +687,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_memory() {
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 	let boot_node = MultiaddrWithPeerId {
 		multiaddr: config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)],
-		peer_id: PeerId::random(),
+		peer_id: PeerId::random().into(),
 	};
 
 	let _ = TestNetworkBuilder::new()
@@ -696,7 +713,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_not_memory() {
 	let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)];
 	let boot_node = MultiaddrWithPeerId {
 		multiaddr: config::build_multiaddr![Memory(rand::random::<u64>())],
-		peer_id: PeerId::random(),
+		peer_id: PeerId::random().into(),
 	};
 
 	let _ = TestNetworkBuilder::new()
@@ -721,7 +738,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_memory() {
 	let listen_addr = config::build_multiaddr![Memory(rand::random::<u64>())];
 	let reserved_node = MultiaddrWithPeerId {
 		multiaddr: config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)],
-		peer_id: PeerId::random(),
+		peer_id: PeerId::random().into(),
 	};
 
 	let _ = TestNetworkBuilder::new()
@@ -750,7 +767,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() {
 	let listen_addr = config::build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(0_u16)];
 	let reserved_node = MultiaddrWithPeerId {
 		multiaddr: config::build_multiaddr![Memory(rand::random::<u64>())],
-		peer_id: PeerId::random(),
+		peer_id: PeerId::random().into(),
 	};
 
 	let _ = TestNetworkBuilder::new()