pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-04-29 04:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Integrate litep2p into Polkadot SDK (#2944)

Browse Source 
[litep2p](https://github.com/altonen/litep2p) is a libp2p-compatible P2P
networking library. It supports all of the features of `rust-libp2p`
that are currently being utilized by Polkadot SDK.

Compared to `rust-libp2p`, `litep2p` has a quite different architecture
which is why the new `litep2p` network backend is only able to use a
little of the existing code in `sc-network`. The design has been mainly
influenced by how we'd wish to structure our networking-related code in
Polkadot SDK: independent higher-levels protocols directly communicating
with the network over links that support bidirectional backpressure. A
good example would be `NotificationHandle`/`RequestResponseHandle`
abstractions which allow, e.g., `SyncingEngine` to directly communicate
with peers to announce/request blocks.

I've tried running `polkadot --network-backend litep2p` with a few
different peer configurations and there is a noticeable reduction in
networking CPU usage. For high load (`--out-peers 200`), networking CPU
usage goes down from ~110% to ~30% (80 pp) and for normal load
(`--out-peers 40`), the usage goes down from ~55% to ~18% (37 pp).

These should not be taken as final numbers because:

a) there are still some low-hanging optimization fruits, such as
enabling [receive window
auto-tuning](https://github.com/libp2p/rust-yamux/pull/176), integrating
`Peerset` more closely with `litep2p` or improving memory usage of the
WebSocket transport
b) fixing bugs/instabilities that incorrectly cause `litep2p` to do less
work will increase the networking CPU usage
c) verification in a more diverse set of tests/conditions is needed

Nevertheless, these numbers should give an early estimate for CPU usage
of the new networking backend.

This PR consists of three separate changes:
* introduce a generic `PeerId` (wrapper around `Multihash`) so that we
don't have use `NetworkService::PeerId` in every part of the code that
uses a `PeerId`
* introduce `NetworkBackend` trait, implement it for the libp2p network
stack and make Polkadot SDK generic over `NetworkBackend`
  * implement `NetworkBackend` for litep2p

The new library should be considered experimental which is why
`rust-libp2p` will remain as the default option for the time being. This
PR currently depends on the master branch of `litep2p` but I'll cut a
new release for the library once all review comments have been
addresses.

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Alexandru Vasile <alexandru.vasile@parity.io>
This commit is contained in:
Aaro Altonen
2024-04-08 19:44:13 +03:00
committed by GitHub
parent 9543d31474
commit 80616f6d03
181 changed files with 11055 additions and 1862 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/network/sync/src/engine.rs
+43 -36
View File
@@ -47,7 +47,7 @@ use futures::{
future::{BoxFuture, Fuse},
FutureExt, StreamExt,
};
use libp2p::{request_response::OutboundFailure, PeerId};
use libp2p::request_response::OutboundFailure;
use log::{debug, error, trace, warn};
use prometheus_endpoint::{
register, Counter, Gauge, MetricSource, Opts, PrometheusError, Registry, SourcedGauge, U64,
@@ -59,21 +59,22 @@ use tokio::time::{Interval, MissedTickBehavior};
use sc_client_api::{BlockBackend, HeaderBackend, ProofProvider};
use sc_consensus::{import_queue::ImportQueueService, IncomingBlock};
use sc_network::{
config::{
FullNetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, NotificationHandshake,
ProtocolId, SetConfig,
},
peer_store::{PeerStoreHandle, PeerStoreProvider},
config::{FullNetworkConfiguration, NotificationHandshake, ProtocolId, SetConfig},
peer_store::PeerStoreProvider,
request_responses::{IfDisconnected, RequestFailure},
service::traits::{Direction, NotificationEvent, ValidationResult},
service::{
traits::{Direction, NotificationConfig, NotificationEvent, ValidationResult},
NotificationMetrics,
},
types::ProtocolName,
utils::LruHashSet,
NotificationService, ReputationChange,
NetworkBackend, NotificationService, ReputationChange,
};
use sc_network_common::{
role::Roles,
sync::message::{BlockAnnounce, BlockAnnouncesHandshake, BlockRequest, BlockState},
};
use sc_network_types::PeerId;
use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
use sp_blockchain::{Error as ClientError, HeaderMetadata};
use sp_consensus::{block_validation::BlockAnnounceValidator, BlockOrigin};
@@ -296,7 +297,7 @@ pub struct SyncingEngine<B: BlockT, Client> {
syncing_started: Option<Instant>,
/// Handle to `PeerStore`.
peer_store_handle: PeerStoreHandle,
peer_store_handle: Arc<dyn PeerStoreProvider>,
/// Instant when the last notification was sent or received.
last_notification_io: Instant,
@@ -328,11 +329,12 @@ where
+ Sync
+ 'static,
{
pub fn new(
pub fn new<N>(
roles: Roles,
client: Arc<Client>,
metrics_registry: Option<&Registry>,
net_config: &FullNetworkConfiguration,
network_metrics: NotificationMetrics,
net_config: &FullNetworkConfiguration<B, <B as BlockT>::Hash, N>,
protocol_id: ProtocolId,
fork_id: &Option<String>,
block_announce_validator: Box<dyn BlockAnnounceValidator<B> + Send>,
@@ -342,8 +344,11 @@ where
block_downloader: Arc<dyn BlockDownloader<B>>,
state_request_protocol_name: ProtocolName,
warp_sync_protocol_name: Option<ProtocolName>,
peer_store_handle: PeerStoreHandle,
) -> Result<(Self, SyncingService<B>, NonDefaultSetConfig), ClientError> {
peer_store_handle: Arc<dyn PeerStoreProvider>,
) -> Result<(Self, SyncingService<B>, N::NotificationProtocolConfig), ClientError>
where
N: NetworkBackend<B, <B as BlockT>::Hash>,
{
let mode = net_config.network_config.sync_mode;
let max_parallel_downloads = net_config.network_config.max_parallel_downloads;
let max_blocks_per_request =
@@ -411,18 +416,22 @@ where
total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize
};
let (block_announce_config, notification_service) = Self::get_block_announce_proto_config(
protocol_id,
fork_id,
roles,
client.info().best_number,
client.info().best_hash,
client
.block_hash(Zero::zero())
.ok()
.flatten()
.expect("Genesis block exists; qed"),
);
let (block_announce_config, notification_service) =
Self::get_block_announce_proto_config::<N>(
protocol_id,
fork_id,
roles,
client.info().best_number,
client.info().best_hash,
client
.block_hash(Zero::zero())
.ok()
.flatten()
.expect("Genesis block exists; qed"),
&net_config.network_config.default_peers_set,
network_metrics,
Arc::clone(&peer_store_handle),
);
// Split warp sync params into warp sync config and a channel to retrieve target block
// header.
@@ -1385,14 +1394,17 @@ where
}
/// Get config for the block announcement protocol
fn get_block_announce_proto_config(
fn get_block_announce_proto_config<N: NetworkBackend<B, <B as BlockT>::Hash>>(
protocol_id: ProtocolId,
fork_id: &Option<String>,
roles: Roles,
best_number: NumberFor<B>,
best_hash: B::Hash,
genesis_hash: B::Hash,
) -> (NonDefaultSetConfig, Box<dyn NotificationService>) {
set_config: &SetConfig,
metrics: NotificationMetrics,
peer_store_handle: Arc<dyn PeerStoreProvider>,
) -> (N::NotificationProtocolConfig, Box<dyn NotificationService>) {
let block_announces_protocol = {
let genesis_hash = genesis_hash.as_ref();
if let Some(ref fork_id) = fork_id {
@@ -1406,7 +1418,7 @@ where
}
};
NonDefaultSetConfig::new(
N::notification_config(
block_announces_protocol.into(),
iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(),
MAX_BLOCK_ANNOUNCE_SIZE,
@@ -1416,14 +1428,9 @@ where
best_hash,
genesis_hash,
))),
// NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement
// protocol is still hardcoded into the peerset.
SetConfig {
in_peers: 0,
out_peers: 0,
reserved_nodes: Vec::new(),
non_reserved_mode: NonReservedPeerMode::Deny,
},
set_config.clone(),
metrics,
peer_store_handle,
)
}