mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-13 23:21:06 +00:00
Alexandru Gheorghe
f9f886886b
## Summary Built on top of the tooling and ideas introduced in https://github.com/paritytech/polkadot-sdk/pull/2528, this PR introduces a synthetic benchmark for measuring and assessing the performance characteristics of the approval-voting and approval-distribution subsystems. Currently this allows, us to simulate the behaviours of these systems based on the following dimensions: ``` TestConfiguration: # Test 1 - objective: !ApprovalsTest last_considered_tranche: 89 min_coalesce: 1 max_coalesce: 6 enable_assignments_v2: true send_till_tranche: 60 stop_when_approved: false coalesce_tranche_diff: 12 workdir_prefix: "/tmp" num_no_shows_per_candidate: 0 approval_distribution_expected_tof: 6.0 approval_distribution_cpu_ms: 3.0 approval_voting_cpu_ms: 4.30 n_validators: 500 n_cores: 100 n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 bandwidth: 524288000000 latency: min_latency: secs: 0 nanos: 1000000 max_latency: secs: 0 nanos: 100000000 error: 0 num_blocks: 10 ``` ## The approach 1. We build a real overseer with the real implementations for approval-voting and approval-distribution subsystems. 2. For a given network size, for each validator we pre-computed all potential assignments and approvals it would send, because this a computation heavy operation this will be cached on a file on disk and be re-used if the generation parameters don't change. 3. The messages will be sent accordingly to the configured parameters and those are split into 3 main benchmarking scenarios. ## Benchmarking scenarios ### Best case scenario *approvals_throughput_best_case.yaml* It send to the approval-distribution only the minimum required tranche to gathered the needed_approvals, so that a candidate is approved. ### Behaviour in the presence of no-shows *approvals_no_shows.yaml* It sends the tranche needed to approve a candidate when we have a maximum of *num_no_shows_per_candidate* tranches with no-shows for each candidate. ### Maximum throughput *approvals_throughput.yaml* It sends all the tranches for each block and measures the used CPU and necessary network bandwidth. by the approval-voting and approval-distribution subsystem. ## How to run it ``` cargo run -p polkadot-subsystem-bench --release -- test-sequence --path polkadot/node/subsystem-bench/examples/approvals_throughput.yaml ``` ## Evaluating performance ### Use the real subsystems metrics If you follow the steps in https://github.com/paritytech/polkadot-sdk/tree/master/polkadot/node/subsystem-bench#install-grafana for installing locally prometheus and grafana, all real metrics for the `approval-distribution`, `approval-voting` and overseer are available. E.g: <img width="2149" alt="Screenshot 2023-12-05 at 11 07 46" src="https://github.com/paritytech/polkadot-sdk/assets/49718502/cb8ae2dd-178b-4922-bfa4-dc37e572ed38"> <img width="2551" alt="Screenshot 2023-12-05 at 11 09 42" src="https://github.com/paritytech/polkadot-sdk/assets/49718502/8b4542ba-88b9-46f9-9b70-cc345366081b"> <img width="2154" alt="Screenshot 2023-12-05 at 11 10 15" src="https://github.com/paritytech/polkadot-sdk/assets/49718502/b8874d8d-632e-443a-9840-14ad8e90c54f"> <img width="2535" alt="Screenshot 2023-12-05 at 11 10 52" src="https://github.com/paritytech/polkadot-sdk/assets/49718502/779a439f-fd18-4985-bb80-85d5afad78e2"> ### Profile with pyroscope 1. Setup pyroscope following the steps in https://github.com/paritytech/polkadot-sdk/tree/master/polkadot/node/subsystem-bench#install-pyroscope, then run any of the benchmark scenario with `--profile` as the arguments. 2. Open the pyroscope dashboard in grafana, e.g: <img width="2544" alt="Screenshot 2024-01-09 at 17 09 58" src="https://github.com/paritytech/polkadot-sdk/assets/49718502/58f50c99-a910-4d20-951a-8b16639303d9"> ### Useful logs 1. Network bandwidth requirements: ``` Payload bytes received from peers: 503993 KiB total, 50399 KiB/block Payload bytes sent to peers: 629971 KiB total, 62997 KiB/block ``` 2. Cpu usage by the approval-distribution/approval-voting subsystems. ``` approval-distribution CPU usage 84.061s approval-distribution CPU usage per block 8.406s approval-voting CPU usage 96.532s approval-voting CPU usage per block 9.653s ``` 3. Time passed until a given block is approved ``` Chain selection approved after 3500 ms hash=0x0101010101010101010101010101010101010101010101010101010101010101 Chain selection approved after 4500 ms hash=0x0202020202020202020202020202020202020202020202020202020202020202 ``` ### Using benchmark to quantify improvements from https://github.com/paritytech/polkadot-sdk/pull/1178 + https://github.com/paritytech/polkadot-sdk/pull/1191 Using a versi-node we compare the scenarios where all new optimisations are disabled with a scenarios where tranche0 assignments are sent in a single message and a conservative simulation where the coalescing of approvals gives us just 50% reduction in the number of messages we send. Overall, what we see is a speedup of around 30-40% in the time it takes to process the necessary messages and a 30-40% reduction in the necessary bandwidth. #### Best case scenario comparison(minimum required tranches sent). Unoptimised ``` Number of blocks: 10 Payload bytes received from peers: 53289 KiB total, 5328 KiB/block Payload bytes sent to peers: 52489 KiB total, 5248 KiB/block approval-distribution CPU usage 6.732s approval-distribution CPU usage per block 0.673s approval-voting CPU usage 9.523s approval-voting CPU usage per block 0.952s ``` vs Optimisation enabled ``` Number of blocks: 10 Payload bytes received from peers: 32141 KiB total, 3214 KiB/block Payload bytes sent to peers: 37314 KiB total, 3731 KiB/block approval-distribution CPU usage 4.658s approval-distribution CPU usage per block 0.466s approval-voting CPU usage 6.236s approval-voting CPU usage per block 0.624s ``` #### Worst case all tranches sent, very unlikely happens when sharding breaks. Unoptimised ``` Number of blocks: 10 Payload bytes received from peers: 746393 KiB total, 74639 KiB/block Payload bytes sent to peers: 729151 KiB total, 72915 KiB/block approval-distribution CPU usage 118.681s approval-distribution CPU usage per block 11.868s approval-voting CPU usage 124.118s approval-voting CPU usage per block 12.412s ``` vs optimised ``` Number of blocks: 10 Payload bytes received from peers: 503993 KiB total, 50399 KiB/block Payload bytes sent to peers: 629971 KiB total, 62997 KiB/block approval-distribution CPU usage 84.061s approval-distribution CPU usage per block 8.406s approval-voting CPU usage 96.532s approval-voting CPU usage per block 9.653s ``` ## TODOs [x] Polish implementation. [x] Use what we have so far to evaluate https://github.com/paritytech/polkadot-sdk/pull/1191 before merging. [x] List of features and additional dimensions we want to use for benchmarking. [x] Run benchmark on hardware similar with versi and kusama nodes. [ ] Add benchmark to be run in CI for catching regression in performance. [ ] Rebase on latest changes for network emulation. --------- Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> Signed-off-by: Alexandru Gheorghe <alexandru.gheorghe@parity.io> Co-authored-by: Andrei Sandu <andrei-mihail@parity.io> Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com>
87 lines
3.6 KiB
TOML
87 lines
3.6 KiB
TOML
[package]
|
|
name = "polkadot-subsystem-bench"
|
|
description = "Subsystem performance benchmark client"
|
|
version = "1.0.0"
|
|
authors.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
readme = "README.md"
|
|
publish = false
|
|
|
|
[[bin]]
|
|
name = "subsystem-bench"
|
|
path = "src/subsystem-bench.rs"
|
|
|
|
# Prevent rustdoc error. Already documented from top-level Cargo.toml.
|
|
doc = false
|
|
|
|
[dependencies]
|
|
polkadot-node-subsystem = { path = "../subsystem" }
|
|
polkadot-node-subsystem-util = { path = "../subsystem-util" }
|
|
polkadot-node-subsystem-types = { path = "../subsystem-types" }
|
|
polkadot-node-primitives = { path = "../primitives" }
|
|
polkadot-primitives = { path = "../../primitives" }
|
|
polkadot-node-network-protocol = { path = "../network/protocol" }
|
|
polkadot-availability-recovery = { path = "../network/availability-recovery", features = ["subsystem-benchmarks"] }
|
|
polkadot-availability-distribution = { path = "../network/availability-distribution" }
|
|
polkadot-node-core-av-store = { path = "../core/av-store" }
|
|
polkadot-node-core-chain-api = { path = "../core/chain-api" }
|
|
polkadot-availability-bitfield-distribution = { path = "../network/bitfield-distribution" }
|
|
color-eyre = { version = "0.6.1", default-features = false }
|
|
polkadot-overseer = { path = "../overseer" }
|
|
colored = "2.0.4"
|
|
assert_matches = "1.5"
|
|
async-trait = "0.1.57"
|
|
sp-keystore = { path = "../../../substrate/primitives/keystore" }
|
|
sc-keystore = { path = "../../../substrate/client/keystore" }
|
|
sp-core = { path = "../../../substrate/primitives/core" }
|
|
clap = { version = "4.4.18", features = ["derive"] }
|
|
futures = "0.3.21"
|
|
futures-timer = "3.0.2"
|
|
bincode = "1.3.3"
|
|
sha1 = "0.10.6"
|
|
hex = "0.4.3"
|
|
gum = { package = "tracing-gum", path = "../gum" }
|
|
polkadot-erasure-coding = { package = "polkadot-erasure-coding", path = "../../erasure-coding" }
|
|
log = "0.4.17"
|
|
env_logger = "0.9.0"
|
|
rand = "0.8.5"
|
|
# `rand` only supports uniform distribution, we need normal distribution for latency.
|
|
rand_distr = "0.4.3"
|
|
bitvec = "1.0.1"
|
|
kvdb-memorydb = "0.13.0"
|
|
|
|
parity-scale-codec = { version = "3.6.1", features = ["derive", "std"] }
|
|
tokio = "1.24.2"
|
|
clap-num = "1.0.2"
|
|
polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
|
|
sp-keyring = { path = "../../../substrate/primitives/keyring" }
|
|
sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" }
|
|
sc-network = { path = "../../../substrate/client/network" }
|
|
sc-service = { path = "../../../substrate/client/service" }
|
|
sp-consensus = { path = "../../../substrate/primitives/consensus/common" }
|
|
polkadot-node-metrics = { path = "../metrics" }
|
|
itertools = "0.11.0"
|
|
polkadot-primitives-test-helpers = { path = "../../primitives/test-helpers" }
|
|
prometheus_endpoint = { package = "substrate-prometheus-endpoint", path = "../../../substrate/utils/prometheus" }
|
|
prometheus = { version = "0.13.0", default-features = false }
|
|
serde = "1.0.195"
|
|
serde_yaml = "0.9"
|
|
|
|
polkadot-node-core-approval-voting = { path = "../core/approval-voting" }
|
|
polkadot-approval-distribution = { path = "../network/approval-distribution" }
|
|
sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" }
|
|
sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false }
|
|
sp-timestamp = { path = "../../../substrate/primitives/timestamp" }
|
|
|
|
schnorrkel = { version = "0.9.1", default-features = false }
|
|
rand_core = "0.6.2" # should match schnorrkel
|
|
rand_chacha = { version = "0.3.1" }
|
|
paste = "1.0.14"
|
|
orchestra = { version = "0.3.5", default-features = false, features = ["futures_channel"] }
|
|
pyroscope = "0.5.7"
|
|
pyroscope_pprofrs = "0.2.7"
|
|
|
|
[features]
|
|
default = []
|