pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-14 09:51:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

subsystem-bench: run cli benchmarks only using config files (#3239)

Browse Source 
This PR removes the configuration of subsystem benchmarks via CLI
arguments. After this, we only keep configurations only in yaml files.
It removes unnecessary code duplication
This commit is contained in:
Andrei Eres
2024-02-08 14:33:32 +01:00
committed by GitHub
parent 84d89e379b
commit 07f8592974
6 changed files with 82 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/node/subsystem-bench/src/subsystem-bench.rs
+76 -150
View File
@@ -17,6 +17,7 @@
//! A tool for running subsystem benchmark tests designed for development and
//! CI regression testing.
use clap::Parser;
use serde::{Deserialize, Serialize};
use colored::Colorize;
@@ -28,24 +29,23 @@ use std::path::Path;
pub(crate) mod approval;
pub(crate) mod availability;
pub(crate) mod cli;
pub(crate) mod core;
mod valgrind;
const LOG_TARGET: &str = "subsystem-bench";
use availability::{prepare_test, NetworkEmulation, TestState};
use cli::TestObjective;
use approval::{bench_approvals, ApprovalsOptions};
use availability::DataAvailabilityReadOptions;
use core::{
configuration::TestConfiguration,
display::display_configuration,
environment::{TestEnvironment, GENESIS_HASH},
};
use clap_num::number_range;
use crate::{approval::bench_approvals, core::display::display_configuration};
fn le_100(s: &str) -> Result<usize, String> {
number_range(s, 0, 100)
}
@@ -54,6 +54,34 @@ fn le_5000(s: &str) -> Result<usize, String> {
number_range(s, 0, 5000)
}
/// Supported test objectives
#[derive(Debug, Clone, Parser, Serialize, Deserialize)]
#[command(rename_all = "kebab-case")]
pub enum TestObjective {
/// Benchmark availability recovery strategies.
DataAvailabilityRead(DataAvailabilityReadOptions),
/// Benchmark availability and bitfield distribution.
DataAvailabilityWrite,
/// Benchmark the approval-voting and approval-distribution subsystems.
ApprovalVoting(ApprovalsOptions),
Unimplemented,
}
impl std::fmt::Display for TestObjective {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
match self {
Self::DataAvailabilityRead(_) => "DataAvailabilityRead",
Self::DataAvailabilityWrite => "DataAvailabilityWrite",
Self::ApprovalVoting(_) => "ApprovalVoting",
Self::Unimplemented => "Unimplemented",
}
)
}
}
#[derive(Debug, Parser)]
#[allow(missing_docs)]
struct BenchCli {
@@ -61,9 +89,6 @@ struct BenchCli {
/// The type of network to be emulated
pub network: NetworkEmulation,
#[clap(flatten)]
pub standard_configuration: cli::StandardTestOptions,
#[clap(short, long)]
/// The bandwidth of emulated remote peers in KiB
pub peer_bandwidth: Option<usize>,
@@ -104,42 +129,12 @@ struct BenchCli {
/// Shows the output in YAML format
pub yaml_output: bool,
#[command(subcommand)]
pub objective: cli::TestObjective,
#[arg(required = true)]
/// Path to the test sequence configuration file
pub path: String,
}
impl BenchCli {
fn create_test_configuration(&self) -> TestConfiguration {
let configuration = &self.standard_configuration;
match self.network {
NetworkEmulation::Healthy => TestConfiguration::healthy_network(
self.objective.clone(),
configuration.num_blocks,
configuration.n_validators,
configuration.n_cores,
configuration.min_pov_size,
configuration.max_pov_size,
),
NetworkEmulation::Degraded => TestConfiguration::degraded_network(
self.objective.clone(),
configuration.num_blocks,
configuration.n_validators,
configuration.n_cores,
configuration.min_pov_size,
configuration.max_pov_size,
),
NetworkEmulation::Ideal => TestConfiguration::ideal_network(
self.objective.clone(),
configuration.num_blocks,
configuration.n_validators,
configuration.n_cores,
configuration.min_pov_size,
configuration.max_pov_size,
),
}
}
fn launch(self) -> eyre::Result<()> {
let is_valgrind_running = valgrind::is_valgrind_running();
if !is_valgrind_running && self.cache_misses {
@@ -156,125 +151,56 @@ impl BenchCli {
None
};
let mut test_config = match self.objective {
TestObjective::TestSequence(options) => {
let test_sequence =
core::configuration::TestSequence::new_from_file(Path::new(&options.path))
.expect("File exists")
.into_vec();
let num_steps = test_sequence.len();
gum::info!(
"{}",
format!("Sequence contains {} step(s)", num_steps).bright_purple()
);
for (index, test_config) in test_sequence.into_iter().enumerate() {
let benchmark_name =
format!("{} #{} {}", &options.path, index + 1, test_config.objective);
gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),);
display_configuration(&test_config);
let test_sequence = core::configuration::TestSequence::new_from_file(Path::new(&self.path))
.expect("File exists")
.into_vec();
let num_steps = test_sequence.len();
gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple());
for (index, test_config) in test_sequence.into_iter().enumerate() {
let benchmark_name = format!("{} #{} {}", &self.path, index + 1, test_config.objective);
gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),);
display_configuration(&test_config);
let usage = match test_config.objective {
TestObjective::DataAvailabilityRead(ref _opts) => {
let mut state = TestState::new(&test_config);
let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
env.runtime().block_on(availability::benchmark_availability_read(
&benchmark_name,
&mut env,
state,
))
},
TestObjective::ApprovalVoting(ref options) => {
let (mut env, state) =
approval::prepare_test(test_config.clone(), options.clone());
env.runtime().block_on(bench_approvals(
&benchmark_name,
&mut env,
state,
))
},
TestObjective::DataAvailabilityWrite => {
let mut state = TestState::new(&test_config);
let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
env.runtime().block_on(availability::benchmark_availability_write(
&benchmark_name,
&mut env,
state,
))
},
TestObjective::TestSequence(_) => todo!(),
TestObjective::Unimplemented => todo!(),
};
let usage = match test_config.objective {
TestObjective::DataAvailabilityRead(ref _opts) => {
let mut state = TestState::new(&test_config);
let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
env.runtime().block_on(availability::benchmark_availability_read(
&benchmark_name,
&mut env,
state,
))
},
TestObjective::ApprovalVoting(ref options) => {
let (mut env, state) =
approval::prepare_test(test_config.clone(), options.clone());
env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state))
},
TestObjective::DataAvailabilityWrite => {
let mut state = TestState::new(&test_config);
let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
env.runtime().block_on(availability::benchmark_availability_write(
&benchmark_name,
&mut env,
state,
))
},
TestObjective::Unimplemented => todo!(),
};
let output = if self.yaml_output {
serde_yaml::to_string(&vec![usage])?
} else {
usage.to_string()
};
println!("{}", output);
}
return Ok(())
},
TestObjective::DataAvailabilityRead(ref _options) => self.create_test_configuration(),
TestObjective::DataAvailabilityWrite => self.create_test_configuration(),
TestObjective::ApprovalVoting(_) => todo!(),
TestObjective::Unimplemented => todo!(),
};
let mut latency_config = test_config.latency.clone().unwrap_or_default();
if let Some(latency) = self.peer_mean_latency {
latency_config.mean_latency_ms = latency;
let output = if self.yaml_output {
serde_yaml::to_string(&vec![usage])?
} else {
usage.to_string()
};
println!("{}", output);
}
if let Some(std_dev) = self.peer_latency_std_dev {
latency_config.std_dev = std_dev;
}
// Write back the updated latency.
test_config.latency = Some(latency_config);
if let Some(connectivity) = self.connectivity {
test_config.connectivity = connectivity;
}
if let Some(bandwidth) = self.peer_bandwidth {
// CLI expects bw in KiB
test_config.peer_bandwidth = bandwidth * 1024;
}
if let Some(bandwidth) = self.bandwidth {
// CLI expects bw in KiB
test_config.bandwidth = bandwidth * 1024;
}
display_configuration(&test_config);
let mut state = TestState::new(&test_config);
let (mut env, _protocol_config) = prepare_test(test_config, &mut state);
let benchmark_name = format!("{}", self.objective);
let usage = match self.objective {
TestObjective::DataAvailabilityRead(_options) => env.runtime().block_on(
availability::benchmark_availability_read(&benchmark_name, &mut env, state),
),
TestObjective::DataAvailabilityWrite => env.runtime().block_on(
availability::benchmark_availability_write(&benchmark_name, &mut env, state),
),
TestObjective::TestSequence(_options) => todo!(),
TestObjective::ApprovalVoting(_) => todo!(),
TestObjective::Unimplemented => todo!(),
};
if let Some(agent_running) = agent_running {
let agent_ready = agent_running.stop()?;
agent_ready.shutdown();
}
let output =
if self.yaml_output { serde_yaml::to_string(&vec![usage])? } else { usage.to_string() };
println!("{}", output);
Ok(())
}
}