pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-05-01 05:27:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Batch benchmarks together with * notation. (#5436)

Browse Source 
* Batch benchmarks together with `*` notation.

* Fix short structopt conflict

* Return error if `batches` is empty

* Move fast benchmarks macro into `frame_benchmarking` (#5445)

* Move macro into `frame_benchmarking`

* Update docs

* Extra line

* Return error if `batches` is empty

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
This commit is contained in:
Gavin Wood
2020-03-31 15:20:09 +02:00
committed by GitHub
parent 954dca689d
commit b472d60a52
4 changed files with 166 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/bin/node/runtime/src/lib.rs
+18 -79
View File
@@ -858,95 +858,34 @@ impl_runtime_apis! {
#[cfg(feature = "runtime-benchmarks")]
impl frame_benchmarking::Benchmark<Block> for Runtime {
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<frame_benchmarking::BenchmarkResults>, sp_runtime::RuntimeString> {
use frame_benchmarking::Benchmarking;
) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
// To get around that, we separated the Session benchmarks into its own crate, which is why
// we need these two lines below.
use pallet_session_benchmarking::Module as SessionBench;
impl pallet_session_benchmarking::Trait for Runtime {}
let result = match module.as_slice() {
b"pallet-balances" | b"balances" => Balances::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-identity" | b"identity" => Identity::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-session" | b"session" => SessionBench::<Runtime>::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-staking" | b"staking" => Staking::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-treasury" | b"treasury" => Treasury::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-vesting" | b"vesting" => Vesting::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-democracy" | b"democracy" => Democracy::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-collective" | b"collective" => Council::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
_ => Err("Benchmark not found for this pallet."),
};
result.map_err(|e| e.into())
let mut batches = Vec::<BenchmarkBatch>::new();
let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
add_benchmark!(params, batches, b"balances", Balances);
add_benchmark!(params, batches, b"im-online", ImOnline);
add_benchmark!(params, batches, b"identity", Identity);
add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
add_benchmark!(params, batches, b"staking", Staking);
add_benchmark!(params, batches, b"timestamp", Timestamp);
add_benchmark!(params, batches, b"treasury", Treasury);
add_benchmark!(params, batches, b"vesting", Vesting);
add_benchmark!(params, batches, b"democracy", Democracy);
add_benchmark!(params, batches, b"collective", Council);
if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
Ok(batches)
}
}
}
substrate/frame/benchmarking/src/lib.rs
+80 -10
View File
@@ -505,15 +505,19 @@ macro_rules! impl_benchmark {
NO_INSTANCE $( $name:ident ),*
) => {
impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}
fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
@@ -597,15 +601,19 @@ macro_rules! impl_benchmark {
INSTANCE $( $name:ident ),*
) => {
impl<T: Trait<I>, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T, I> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}
fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
@@ -686,3 +694,65 @@ macro_rules! impl_benchmark {
}
}
}
/// This macro adds pallet benchmarks to a `Vec<BenchmarkBatch>` object.
///
/// First create an object that holds in the input parameters for the benchmark:
///
/// ```ignore
/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
/// ```
///
/// Then define a mutable local variable to hold your `BenchmarkBatch` object:
///
/// ```ignore
/// let mut batches = Vec::<BenchmarkBatch>::new();
/// ````
///
/// Then add the pallets you want to benchmark to this object, including the string
/// you want to use target a particular pallet:
///
/// ```ignore
/// add_benchmark!(params, batches, b"balances", Balances);
/// add_benchmark!(params, batches, b"identity", Identity);
/// add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
/// ...
/// ```
///
/// At the end of `dispatch_benchmark`, you should return this batches object.
#[macro_export]
macro_rules! add_benchmark {
( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params;
if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
for benchmark in $( $location )*::benchmarks().into_iter() {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
benchmark,
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.to_vec(),
});
}
} else {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
&benchmark[..],
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.clone(),
});
}
}
)
}
substrate/frame/benchmarking/src/utils.rs
+25 -9
View File
@@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256;
use sp_runtime::RuntimeString;
/// An alphabet of possible parameters to use for benchmarking.
#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)]
#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)]
#[allow(missing_docs)]
#[allow(non_camel_case_types)]
pub enum BenchmarkParameter {
a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
}
/// The results of a single of benchmark.
#[derive(Encode, Decode, Clone, PartialEq, Debug)]
pub struct BenchmarkBatch {
/// The pallet containing this benchmark.
pub pallet: Vec<u8>,
/// The extrinsic (or benchmark name) of this benchmark.
pub benchmark: Vec<u8>,
/// The results from this benchmark.
pub results: Vec<BenchmarkResults>,
}
/// Results from running benchmarks on a FRAME pallet.
/// Contains duration of the function call in nanoseconds along with the benchmark parameters
/// used for that benchmark result.
@@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! {
pub trait Benchmark {
/// Dispatch the given benchmark.
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<BenchmarkResults>, RuntimeString>;
) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
}
}
@@ -75,19 +86,24 @@ pub trait Benchmarking {
/// The pallet benchmarking trait.
pub trait Benchmarking<T> {
/// Get the benchmarks available for this pallet. Generally there is one benchmark per
/// extrinsic, so these are sometimes just called "extrinsics".
fn benchmarks() -> Vec<&'static [u8]>;
/// Run the benchmarks for this pallet.
///
/// Parameters
/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
/// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as
/// bytes.
/// - `steps`: The number of sample points you want to take across the range of parameters.
/// - `lowest_range_values`: The lowest number for each range of parameters.
/// - `highest_range_values`: The highest number for each range of parameters.
/// - `repeat`: The number of times you want to repeat a benchmark.
fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
name: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<T>, &'static str>;
}
substrate/utils/frame/benchmarking-cli/src/lib.rs
+43 -28
View File
@@ -22,7 +22,7 @@ use sc_client_db::BenchmarkingState;
use sc_service::{Configuration, ChainSpec};
use sc_executor::{NativeExecutor, NativeExecutionDispatch};
use codec::{Encode, Decode};
use frame_benchmarking::{BenchmarkResults, Analysis};
use frame_benchmarking::{BenchmarkBatch, Analysis};
use sp_core::{
tasks,
traits::KeystoreExt,
@@ -33,11 +33,11 @@ use sp_externalities::Extensions;
/// The `benchmark` command used to benchmark FRAME Pallets.
#[derive(Debug, structopt::StructOpt, Clone)]
pub struct BenchmarkCmd {
/// Select a FRAME Pallet to benchmark.
/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
#[structopt(short, long)]
pub pallet: String,
/// Select an extrinsic to benchmark.
/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
#[structopt(short, long)]
pub extrinsic: String,
@@ -46,17 +46,29 @@ pub struct BenchmarkCmd {
pub steps: Vec<u32>,
/// Indicates lowest values for each of the component ranges.
#[structopt(long, use_delimiter = true)]
#[structopt(long = "low", use_delimiter = true)]
pub lowest_range_values: Vec<u32>,
/// Indicates highest values for each of the component ranges.
#[structopt(long, use_delimiter = true)]
#[structopt(long = "high", use_delimiter = true)]
pub highest_range_values: Vec<u32>,
/// Select how many repetitions of this benchmark should run.
#[structopt(short, long, default_value = "1")]
pub repeat: u32,
/// Print the raw results.
#[structopt(long = "raw")]
pub raw_data: bool,
/// Don't print the median-slopes linear regression analysis.
#[structopt(long)]
pub no_median_slopes: bool,
/// Don't print the min-squares linear regression analysis.
#[structopt(long)]
pub no_min_squares: bool,
#[allow(missing_docs)]
#[structopt(flatten)]
pub shared_params: sc_cli::SharedParams,
@@ -135,47 +147,50 @@ impl BenchmarkCmd {
.execute(strategy.into())
.map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;
let results = <Result<Vec<BenchmarkResults>, String> as Decode>::decode(&mut &result[..])
let results = <Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
match results {
Ok(results) => {
Ok(batches) => for batch in batches.into_iter() {
// Print benchmark metadata
println!(
"Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}",
self.pallet,
self.extrinsic,
String::from_utf8(batch.pallet).expect("Encoded from String; qed"),
String::from_utf8(batch.benchmark).expect("Encoded from String; qed"),
self.lowest_range_values,
self.highest_range_values,
self.steps,
self.repeat,
);
// Print the table header
results[0].0.iter().for_each(|param| print!("{:?},", param.0));
if self.raw_data {
// Print the table header
batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0));
print!("extrinsic_time,storage_root_time\n");
// Print the values
results.iter().for_each(|result| {
let parameters = &result.0;
parameters.iter().for_each(|param| print!("{:?},", param.1));
// Print extrinsic time and storage root time
print!("{:?},{:?}\n", result.1, result.2);
});
print!("extrinsic_time,storage_root_time\n");
// Print the values
batch.results.iter().for_each(|result| {
let parameters = &result.0;
parameters.iter().for_each(|param| print!("{:?},", param.1));
// Print extrinsic time and storage root time
print!("{:?},{:?}\n", result.1, result.2);
});
print!("\n");
print!("\n");
}
// Conduct analysis.
if let Some(analysis) = Analysis::median_slopes(&results) {
println!("Median Slopes Analysis\n========\n{}", analysis);
if !self.no_median_slopes {
if let Some(analysis) = Analysis::median_slopes(&batch.results) {
println!("Median Slopes Analysis\n========\n{}", analysis);
}
}
if let Some(analysis) = Analysis::min_squares_iqr(&results) {
println!("Min Squares Analysis\n========\n{}", analysis);
if !self.no_min_squares {
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) {
println!("Min Squares Analysis\n========\n{}", analysis);
}
}
eprintln!("Done.");
}
},
Err(error) => eprintln!("Error: {:?}", error),
}