pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-20 19:45:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate litep2p into Polkadot SDK (#2944)

Browse Source 
[litep2p](https://github.com/altonen/litep2p) is a libp2p-compatible P2P
networking library. It supports all of the features of `rust-libp2p`
that are currently being utilized by Polkadot SDK.

Compared to `rust-libp2p`, `litep2p` has a quite different architecture
which is why the new `litep2p` network backend is only able to use a
little of the existing code in `sc-network`. The design has been mainly
influenced by how we'd wish to structure our networking-related code in
Polkadot SDK: independent higher-levels protocols directly communicating
with the network over links that support bidirectional backpressure. A
good example would be `NotificationHandle`/`RequestResponseHandle`
abstractions which allow, e.g., `SyncingEngine` to directly communicate
with peers to announce/request blocks.

I've tried running `polkadot --network-backend litep2p` with a few
different peer configurations and there is a noticeable reduction in
networking CPU usage. For high load (`--out-peers 200`), networking CPU
usage goes down from ~110% to ~30% (80 pp) and for normal load
(`--out-peers 40`), the usage goes down from ~55% to ~18% (37 pp).

These should not be taken as final numbers because:

a) there are still some low-hanging optimization fruits, such as
enabling [receive window
auto-tuning](https://github.com/libp2p/rust-yamux/pull/176), integrating
`Peerset` more closely with `litep2p` or improving memory usage of the
WebSocket transport
b) fixing bugs/instabilities that incorrectly cause `litep2p` to do less
work will increase the networking CPU usage
c) verification in a more diverse set of tests/conditions is needed

Nevertheless, these numbers should give an early estimate for CPU usage
of the new networking backend.

This PR consists of three separate changes:
* introduce a generic `PeerId` (wrapper around `Multihash`) so that we
don't have use `NetworkService::PeerId` in every part of the code that
uses a `PeerId`
* introduce `NetworkBackend` trait, implement it for the libp2p network
stack and make Polkadot SDK generic over `NetworkBackend`
  * implement `NetworkBackend` for litep2p

The new library should be considered experimental which is why
`rust-libp2p` will remain as the default option for the time being. This
PR currently depends on the master branch of `litep2p` but I'll cut a
new release for the library once all review comments have been
addresses.

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Alexandru Vasile <alexandru.vasile@parity.io>
This commit is contained in:
Aaro Altonen
2024-04-08 19:44:13 +03:00
committed by GitHub
parent 9543d31474
commit 80616f6d03
181 changed files with 11055 additions and 1862 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cumulus/test/service/src/lib.rs
+40 -21
View File
@@ -62,7 +62,9 @@ use polkadot_service::ProvideRuntimeApi;
use sc_consensus::ImportQueue;
use sc_network::{
config::{FullNetworkConfiguration, TransportConfig},
multiaddr, NetworkBlock, NetworkService, NetworkStateInfo,
multiaddr,
service::traits::NetworkService,
NetworkBackend, NetworkBlock, NetworkStateInfo,
};
use sc_service::{
config::{
@@ -74,7 +76,7 @@ use sc_service::{
};
use sp_arithmetic::traits::SaturatedConversion;
use sp_blockchain::HeaderBackend;
use sp_core::{Pair, H256};
use sp_core::Pair;
use sp_keyring::Sr25519Keyring;
use sp_runtime::{codec::Encode, generic};
use sp_state_machine::BasicExternalities;
@@ -304,7 +306,7 @@ async fn build_relay_chain_interface(
///
/// This is the actual implementation that is abstract over the executor and the runtime api.
#[sc_tracing::logging::prefix_logs_with(parachain_config.network.node_name.as_str())]
pub async fn start_node_impl<RB>(
pub async fn start_node_impl<RB, Net: NetworkBackend<Block, Hash>>(
parachain_config: Configuration,
collator_key: Option<CollatorPair>,
relay_chain_config: Configuration,
@@ -318,7 +320,7 @@ pub async fn start_node_impl<RB>(
) -> sc_service::error::Result<(
TaskManager,
Arc<Client>,
Arc<NetworkService<Block, H256>>,
Arc<dyn NetworkService>,
RpcHandlers,
TransactionPool,
Arc<Backend>,
@@ -348,7 +350,7 @@ where
.map_err(|e| sc_service::Error::Application(Box::new(e) as Box<_>))?;
let import_queue_service = params.import_queue.service();
let net_config = FullNetworkConfiguration::new(&parachain_config.network);
let net_config = FullNetworkConfiguration::<Block, Hash, Net>::new(&parachain_config.network);
let (network, system_rpc_tx, tx_handler_controller, start_network, sync_service) =
build_network(BuildNetworkParams {
@@ -494,7 +496,7 @@ pub struct TestNode {
/// Client's instance.
pub client: Arc<Client>,
/// Node's network.
pub network: Arc<NetworkService<Block, H256>>,
pub network: Arc<dyn NetworkService>,
/// The `MultiaddrWithPeerId` to this node. This is useful if you want to pass it as "boot
/// node" to other nodes.
pub addr: MultiaddrWithPeerId,
@@ -702,21 +704,38 @@ impl TestNodeBuilder {
let multiaddr = parachain_config.network.listen_addresses[0].clone();
let (task_manager, client, network, rpc_handlers, transaction_pool, backend) =
start_node_impl(
parachain_config,
self.collator_key,
relay_chain_config,
self.para_id,
self.wrap_announce_block,
false,
|_| Ok(jsonrpsee::RpcModule::new(())),
self.consensus,
collator_options,
self.record_proof_during_import,
)
.await
.expect("could not create Cumulus test service");
match relay_chain_config.network.network_backend {
sc_network::config::NetworkBackendType::Libp2p =>
start_node_impl::<_, sc_network::NetworkWorker<_, _>>(
parachain_config,
self.collator_key,
relay_chain_config,
self.para_id,
self.wrap_announce_block,
false,
|_| Ok(jsonrpsee::RpcModule::new(())),
self.consensus,
collator_options,
self.record_proof_during_import,
)
.await
.expect("could not create Cumulus test service"),
sc_network::config::NetworkBackendType::Litep2p =>
start_node_impl::<_, sc_network::Litep2pNetworkBackend>(
parachain_config,
self.collator_key,
relay_chain_config,
self.para_id,
self.wrap_announce_block,
false,
|_| Ok(jsonrpsee::RpcModule::new(())),
self.consensus,
collator_options,
self.record_proof_during_import,
)
.await
.expect("could not create Cumulus test service"),
};
let peer_id = network.local_peer_id();
let addr = MultiaddrWithPeerId { multiaddr, peer_id };
cumulus/test/service/src/main.rs
+38 -12
View File
@@ -101,18 +101,44 @@ fn main() -> Result<(), sc_cli::Error> {
.unwrap_or(cumulus_test_service::Consensus::RelayChain);
let (mut task_manager, _, _, _, _, _) = tokio_runtime
.block_on(cumulus_test_service::start_node_impl(
config,
collator_key,
polkadot_config,
parachain_id.into(),
cli.disable_block_announcements.then(wrap_announce_block),
cli.fail_pov_recovery,
|_| Ok(jsonrpsee::RpcModule::new(())),
consensus,
collator_options,
true,
))
.block_on(async move {
match polkadot_config.network.network_backend {
sc_network::config::NetworkBackendType::Libp2p =>
cumulus_test_service::start_node_impl::<
_,
sc_network::NetworkWorker<_, _>,
>(
config,
collator_key,
polkadot_config,
parachain_id.into(),
cli.disable_block_announcements.then(wrap_announce_block),
cli.fail_pov_recovery,
|_| Ok(jsonrpsee::RpcModule::new(())),
consensus,
collator_options,
true,
)
.await,
sc_network::config::NetworkBackendType::Litep2p =>
cumulus_test_service::start_node_impl::<
_,
sc_network::Litep2pNetworkBackend,
>(
config,
collator_key,
polkadot_config,
parachain_id.into(),
cli.disable_block_announcements.then(wrap_announce_block),
cli.fail_pov_recovery,
|_| Ok(jsonrpsee::RpcModule::new(())),
consensus,
collator_options,
true,
)
.await,
}
})
.expect("could not create Cumulus test service");
tokio_runtime