Files
pezkuwi-subxt/substrate/client/cli/src/params/network_params.rs
T
Aaro Altonen 80616f6d03 Integrate litep2p into Polkadot SDK (#2944)
[litep2p](https://github.com/altonen/litep2p) is a libp2p-compatible P2P
networking library. It supports all of the features of `rust-libp2p`
that are currently being utilized by Polkadot SDK.

Compared to `rust-libp2p`, `litep2p` has a quite different architecture
which is why the new `litep2p` network backend is only able to use a
little of the existing code in `sc-network`. The design has been mainly
influenced by how we'd wish to structure our networking-related code in
Polkadot SDK: independent higher-levels protocols directly communicating
with the network over links that support bidirectional backpressure. A
good example would be `NotificationHandle`/`RequestResponseHandle`
abstractions which allow, e.g., `SyncingEngine` to directly communicate
with peers to announce/request blocks.

I've tried running `polkadot --network-backend litep2p` with a few
different peer configurations and there is a noticeable reduction in
networking CPU usage. For high load (`--out-peers 200`), networking CPU
usage goes down from ~110% to ~30% (80 pp) and for normal load
(`--out-peers 40`), the usage goes down from ~55% to ~18% (37 pp).

These should not be taken as final numbers because:

a) there are still some low-hanging optimization fruits, such as
enabling [receive window
auto-tuning](https://github.com/libp2p/rust-yamux/pull/176), integrating
`Peerset` more closely with `litep2p` or improving memory usage of the
WebSocket transport
b) fixing bugs/instabilities that incorrectly cause `litep2p` to do less
work will increase the networking CPU usage
c) verification in a more diverse set of tests/conditions is needed

Nevertheless, these numbers should give an early estimate for CPU usage
of the new networking backend.

This PR consists of three separate changes:
* introduce a generic `PeerId` (wrapper around `Multihash`) so that we
don't have use `NetworkService::PeerId` in every part of the code that
uses a `PeerId`
* introduce `NetworkBackend` trait, implement it for the libp2p network
stack and make Polkadot SDK generic over `NetworkBackend`
  * implement `NetworkBackend` for litep2p

The new library should be considered experimental which is why
`rust-libp2p` will remain as the default option for the time being. This
PR currently depends on the master branch of `litep2p` but I'll cut a
new release for the library once all review comments have been
addresses.

---------

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>
Co-authored-by: Dmitry Markin <dmitry@markin.tech>
Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Alexandru Vasile <alexandru.vasile@parity.io>
2024-04-08 16:44:13 +00:00

337 lines
11 KiB
Rust

// This file is part of Substrate.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
arg_enums::{NetworkBackendType, SyncMode},
params::node_key_params::NodeKeyParams,
};
use clap::Args;
use sc_network::{
config::{
NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig,
},
multiaddr::Protocol,
};
use sc_service::{
config::{Multiaddr, MultiaddrWithPeerId},
ChainSpec, ChainType,
};
use std::{borrow::Cow, num::NonZeroUsize, path::PathBuf};
/// Parameters used to create the network configuration.
#[derive(Debug, Clone, Args)]
pub struct NetworkParams {
/// Specify a list of bootnodes.
#[arg(long, value_name = "ADDR", num_args = 1..)]
pub bootnodes: Vec<MultiaddrWithPeerId>,
/// Specify a list of reserved node addresses.
#[arg(long, value_name = "ADDR", num_args = 1..)]
pub reserved_nodes: Vec<MultiaddrWithPeerId>,
/// Whether to only synchronize the chain with reserved nodes.
///
/// Also disables automatic peer discovery.
/// TCP connections might still be established with non-reserved nodes.
/// In particular, if you are a validator your node might still connect to other
/// validator nodes and collator nodes regardless of whether they are defined as
/// reserved nodes.
#[arg(long)]
pub reserved_only: bool,
/// Public address that other nodes will use to connect to this node.
///
/// This can be used if there's a proxy in front of this node.
#[arg(long, value_name = "PUBLIC_ADDR", num_args = 1..)]
pub public_addr: Vec<Multiaddr>,
/// Listen on this multiaddress.
///
/// By default:
/// If `--validator` is passed: `/ip4/0.0.0.0/tcp/<port>` and `/ip6/[::]/tcp/<port>`.
/// Otherwise: `/ip4/0.0.0.0/tcp/<port>/ws` and `/ip6/[::]/tcp/<port>/ws`.
#[arg(long, value_name = "LISTEN_ADDR", num_args = 1..)]
pub listen_addr: Vec<Multiaddr>,
/// Specify p2p protocol TCP port.
#[arg(long, value_name = "PORT", conflicts_with_all = &[ "listen_addr" ])]
pub port: Option<u16>,
/// Always forbid connecting to private IPv4/IPv6 addresses.
///
/// The option doesn't apply to addresses passed with `--reserved-nodes` or
/// `--bootnodes`. Enabled by default for chains marked as "live" in their chain
/// specifications.
///
/// Address allocation for private networks is specified by
/// [RFC1918](https://tools.ietf.org/html/rfc1918)).
#[arg(long, alias = "no-private-ipv4", conflicts_with_all = &["allow_private_ip"])]
pub no_private_ip: bool,
/// Always accept connecting to private IPv4/IPv6 addresses.
///
/// Enabled by default for chains marked as "local" in their chain specifications,
/// or when `--dev` is passed.
///
/// Address allocation for private networks is specified by
/// [RFC1918](https://tools.ietf.org/html/rfc1918)).
#[arg(long, alias = "allow-private-ipv4", conflicts_with_all = &["no_private_ip"])]
pub allow_private_ip: bool,
/// Number of outgoing connections we're trying to maintain.
#[arg(long, value_name = "COUNT", default_value_t = 8)]
pub out_peers: u32,
/// Maximum number of inbound full nodes peers.
#[arg(long, value_name = "COUNT", default_value_t = 32)]
pub in_peers: u32,
/// Maximum number of inbound light nodes peers.
#[arg(long, value_name = "COUNT", default_value_t = 100)]
pub in_peers_light: u32,
/// Disable mDNS discovery (default: true).
///
/// By default, the network will use mDNS to discover other nodes on the
/// local network. This disables it. Automatically implied when using --dev.
#[arg(long)]
pub no_mdns: bool,
/// Maximum number of peers from which to ask for the same blocks in parallel.
///
/// This allows downloading announced blocks from multiple peers.
/// Decrease to save traffic and risk increased latency.
#[arg(long, value_name = "COUNT", default_value_t = 5)]
pub max_parallel_downloads: u32,
#[allow(missing_docs)]
#[clap(flatten)]
pub node_key_params: NodeKeyParams,
/// Enable peer discovery on local networks.
///
/// By default this option is `true` for `--dev` or when the chain type is
/// `Local`/`Development` and false otherwise.
#[arg(long)]
pub discover_local: bool,
/// Require iterative Kademlia DHT queries to use disjoint paths.
///
/// Disjoint paths increase resiliency in the presence of potentially adversarial nodes.
///
/// See the S/Kademlia paper for more information on the high level design as well as its
/// security improvements.
#[arg(long)]
pub kademlia_disjoint_query_paths: bool,
/// Kademlia replication factor.
///
/// Determines to how many closest peers a record is replicated to.
///
/// Discovery mechanism requires successful replication to all
/// `kademlia_replication_factor` peers to consider record successfully put.
#[arg(long, default_value = "20")]
pub kademlia_replication_factor: NonZeroUsize,
/// Join the IPFS network and serve transactions over bitswap protocol.
#[arg(long)]
pub ipfs_server: bool,
/// Blockchain syncing mode.
#[arg(
long,
value_enum,
value_name = "SYNC_MODE",
default_value_t = SyncMode::Full,
ignore_case = true,
verbatim_doc_comment
)]
pub sync: SyncMode,
/// Maximum number of blocks per request.
///
/// Try reducing this number from the default value if you have a slow network connection
/// and observe block requests timing out.
#[arg(long, value_name = "COUNT", default_value_t = 64)]
pub max_blocks_per_request: u32,
/// Network backend used for P2P networking.
///
/// litep2p network backend is considered experimental and isn't as stable as the libp2p
/// network backend.
#[arg(
long,
value_enum,
value_name = "NETWORK_BACKEND",
default_value_t = NetworkBackendType::Libp2p,
ignore_case = true,
verbatim_doc_comment
)]
pub network_backend: NetworkBackendType,
}
impl NetworkParams {
/// Fill the given `NetworkConfiguration` by looking at the cli parameters.
pub fn network_config(
&self,
chain_spec: &Box<dyn ChainSpec>,
is_dev: bool,
is_validator: bool,
net_config_path: Option<PathBuf>,
client_id: &str,
node_name: &str,
node_key: NodeKeyConfig,
default_listen_port: u16,
) -> NetworkConfiguration {
let port = self.port.unwrap_or(default_listen_port);
let listen_addresses = if self.listen_addr.is_empty() {
if is_validator || is_dev {
vec![
Multiaddr::empty()
.with(Protocol::Ip6([0, 0, 0, 0, 0, 0, 0, 0].into()))
.with(Protocol::Tcp(port)),
Multiaddr::empty()
.with(Protocol::Ip4([0, 0, 0, 0].into()))
.with(Protocol::Tcp(port)),
]
} else {
vec![
Multiaddr::empty()
.with(Protocol::Ip6([0, 0, 0, 0, 0, 0, 0, 0].into()))
.with(Protocol::Tcp(port))
.with(Protocol::Ws(Cow::Borrowed("/"))),
Multiaddr::empty()
.with(Protocol::Ip4([0, 0, 0, 0].into()))
.with(Protocol::Tcp(port))
.with(Protocol::Ws(Cow::Borrowed("/"))),
]
}
} else {
self.listen_addr.clone()
};
let public_addresses = self.public_addr.clone();
let mut boot_nodes = chain_spec.boot_nodes().to_vec();
boot_nodes.extend(self.bootnodes.clone());
let chain_type = chain_spec.chain_type();
// Activate if the user explicitly requested local discovery, `--dev` is given or the
// chain type is `Local`/`Development`
let allow_non_globals_in_dht =
self.discover_local ||
is_dev || matches!(chain_type, ChainType::Local | ChainType::Development);
let allow_private_ip = match (self.allow_private_ip, self.no_private_ip) {
(true, true) => unreachable!("`*_private_ip` flags are mutually exclusive; qed"),
(true, false) => true,
(false, true) => false,
(false, false) =>
is_dev || matches!(chain_type, ChainType::Local | ChainType::Development),
};
NetworkConfiguration {
boot_nodes,
net_config_path,
default_peers_set: SetConfig {
in_peers: self.in_peers + self.in_peers_light,
out_peers: self.out_peers,
reserved_nodes: self.reserved_nodes.clone(),
non_reserved_mode: if self.reserved_only {
NonReservedPeerMode::Deny
} else {
NonReservedPeerMode::Accept
},
},
default_peers_set_num_full: self.in_peers + self.out_peers,
listen_addresses,
public_addresses,
node_key,
node_name: node_name.to_string(),
client_version: client_id.to_string(),
transport: TransportConfig::Normal {
enable_mdns: !is_dev && !self.no_mdns,
allow_private_ip,
},
max_parallel_downloads: self.max_parallel_downloads,
max_blocks_per_request: self.max_blocks_per_request,
enable_dht_random_walk: !self.reserved_only,
allow_non_globals_in_dht,
kademlia_disjoint_query_paths: self.kademlia_disjoint_query_paths,
kademlia_replication_factor: self.kademlia_replication_factor,
yamux_window_size: None,
ipfs_server: self.ipfs_server,
sync_mode: self.sync.into(),
network_backend: self.network_backend.into(),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
#[derive(Parser)]
struct Cli {
#[clap(flatten)]
network_params: NetworkParams,
}
#[test]
fn reserved_nodes_multiple_values_and_occurrences() {
let params = Cli::try_parse_from([
"",
"--reserved-nodes",
"/ip4/0.0.0.0/tcp/501/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS",
"/ip4/0.0.0.0/tcp/502/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS",
"--reserved-nodes",
"/ip4/0.0.0.0/tcp/503/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS",
])
.expect("Parses network params");
let expected = vec![
MultiaddrWithPeerId::try_from(
"/ip4/0.0.0.0/tcp/501/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS"
.to_string(),
)
.unwrap(),
MultiaddrWithPeerId::try_from(
"/ip4/0.0.0.0/tcp/502/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS"
.to_string(),
)
.unwrap(),
MultiaddrWithPeerId::try_from(
"/ip4/0.0.0.0/tcp/503/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS"
.to_string(),
)
.unwrap(),
];
assert_eq!(expected, params.network_params.reserved_nodes);
}
#[test]
fn sync_ignores_case() {
let params = Cli::try_parse_from(["", "--sync", "wArP"]).expect("Parses network params");
assert_eq!(SyncMode::Warp, params.network_params.sync);
}
}