pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-14 04:01:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable verification logic when executing benchmarks (#6929)

Browse Source 
* Add `--verify` flag to benchmark execution

* make it so `--verify` can be used for getting the actual benchmarks

* undo manual testing

* oops

* use benchmark config struct

* verify is default on, docs update

* remove clone

* improve formatting

* fix test

* bump impl for ci
This commit is contained in:
Shawn Tabrizi
2020-08-24 15:24:00 +02:00
committed by GitHub
parent c8ca2ce183
commit 4462f7150d
7 changed files with 150 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/benchmarking/src/lib.rs
+92 -87
View File
@@ -648,9 +648,11 @@ macro_rules! benchmark_backend {
]
}
fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
{
fn instance(
&self,
components: &[($crate::BenchmarkParameter, u32)],
verify: bool
) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
$(
let $common = $common_from;
)*
@@ -666,28 +668,13 @@ macro_rules! benchmark_backend {
$( $param_instancer ; )*
$( $post )*
Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) }))
}
fn verify(&self, components: &[($crate::BenchmarkParameter, u32)])
-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
{
$(
let $common = $common_from;
)*
$(
// Prepare instance
let $param = components.iter()
.find(|&c| c.0 == $crate::BenchmarkParameter::$param)
.unwrap().1;
)*
$(
let $pre_id : $pre_ty = $pre_ex;
)*
$( $param_instancer ; )*
$( $post )*
Ok(Box::new(move || -> Result<(), &'static str> { $eval; $postcode; Ok(()) }))
Ok(Box::new(move || -> Result<(), &'static str> {
$eval;
if verify {
$postcode;
}
Ok(())
}))
}
}
};
@@ -736,26 +723,16 @@ macro_rules! selected_benchmark {
}
}
fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
{
fn instance(
&self,
components: &[($crate::BenchmarkParameter, u32)],
verify: bool
) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
match self {
$(
Self::$bench => <
$bench as $crate::BenchmarkingSetup<T $(, $bench_inst)? >
>::instance(&$bench, components),
)*
}
}
fn verify(&self, components: &[($crate::BenchmarkParameter, u32)])
-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
{
match self {
$(
Self::$bench => <
$bench as $crate::BenchmarkingSetup<T $(, $bench_inst)? >
>::verify(&$bench, components),
>::instance(&$bench, components, verify),
)*
}
}
@@ -791,7 +768,8 @@ macro_rules! impl_benchmark {
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
whitelist: &[$crate::TrackedStorageKey]
whitelist: &[$crate::TrackedStorageKey],
verify: bool,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -826,6 +804,7 @@ macro_rules! impl_benchmark {
repeat: u32,
c: Vec<($crate::BenchmarkParameter, u32)>,
results: &mut Vec<$crate::BenchmarkResults>,
verify: bool,
| -> Result<(), &'static str> {
// Run the benchmark `repeat` times.
for _ in 0..repeat {
@@ -833,7 +812,7 @@ macro_rules! impl_benchmark {
// benchmark.
let closure_to_benchmark = <
SelectedBenchmark as $crate::BenchmarkingSetup<T $(, $instance)?>
>::instance(&selected_benchmark, &c)?;
>::instance(&selected_benchmark, &c, verify)?;
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
@@ -847,43 +826,49 @@ macro_rules! impl_benchmark {
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?}", c
);
if verify {
closure_to_benchmark()?;
} else {
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?}", c
);
let start_extrinsic = $crate::benchmarking::current_time();
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
let start_extrinsic = $crate::benchmarking::current_time();
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
closure_to_benchmark()?;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
}
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
@@ -893,7 +878,11 @@ macro_rules! impl_benchmark {
};
if components.is_empty() {
repeat_benchmark(repeat, Default::default(), &mut results)?;
if verify {
// If `--verify` is used, run the benchmark once to verify it would complete.
repeat_benchmark(1, Default::default(), &mut Vec::new(), true)?;
}
repeat_benchmark(repeat, Default::default(), &mut results, false)?;
} else {
// Select the component we will be benchmarking. Each component will be benchmarked.
for (idx, (name, low, high)) in components.iter().enumerate() {
@@ -929,7 +918,11 @@ macro_rules! impl_benchmark {
)
.collect();
repeat_benchmark(repeat, c, &mut results)?;
if verify {
// If `--verify` is used, run the benchmark once to verify it would complete.
repeat_benchmark(1, Default::default(), &mut Vec::new(), true)?;
}
repeat_benchmark(repeat, c, &mut results, false)?;
}
}
}
@@ -962,17 +955,17 @@ macro_rules! impl_benchmark_test {
let execute_benchmark = |
c: Vec<($crate::BenchmarkParameter, u32)>
| -> Result<(), &'static str> {
// Set up the verification state
// Set up the benchmark, return execution + verification function.
let closure_to_verify = <
SelectedBenchmark as $crate::BenchmarkingSetup<T, _>
>::verify(&selected_benchmark, &c)?;
>::instance(&selected_benchmark, &c, true)?;
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Run verification
// Run execution + verification
closure_to_verify()?;
// Reset the state
@@ -1015,7 +1008,7 @@ macro_rules! impl_benchmark_test {
/// First create an object that holds in the input parameters for the benchmark:
///
/// ```ignore
/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist);
/// let params = (&config, &whitelist);
/// ```
///
/// The `whitelist` is a parameter you pass to control the DB read/write tracking.
@@ -1059,18 +1052,29 @@ macro_rules! impl_benchmark_test {
macro_rules! add_benchmark {
( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => (
let name_string = stringify!($name).as_bytes();
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist, extra) = $params;
let (config, whitelist) = $params;
let $crate::BenchmarkConfig {
pallet,
benchmark,
lowest_range_values,
highest_range_values,
steps,
repeat,
verify,
extra,
} = config;
if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] {
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
for benchmark in $( $location )*::benchmarks(extra).into_iter() {
for benchmark in $( $location )*::benchmarks(*extra).into_iter() {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
benchmark,
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
*repeat,
whitelist,
*verify,
)?,
pallet: name_string.to_vec(),
benchmark: benchmark.to_vec(),
@@ -1083,8 +1087,9 @@ macro_rules! add_benchmark {
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
*repeat,
whitelist,
*verify,
)?,
pallet: name_string.to_vec(),
benchmark: benchmark.clone(),