pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 17:01:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test each benchmark case in own #[test] (#9860)

Browse Source 
* Generate one #[test] fn per bench case.

* Update benchmark macro syntax in frame pallets.

* Explain new benchmark macro syntax in example pallet.

* support with and without a semicolon

* update pallets to use individual tests

* migrate staking too

* migrate more pallets

* fix up democracy and use individual tests

* Fix comment

* Put println message in panic

* Remove `another_set_dummy` from doc

`another_set_dummy` is not present in the benchmarking.rs (anymore).

* Update doc for benchmarks macro

* Update doc for impl_benchmark_test_suite macro

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
This commit is contained in:
ucover
2021-10-01 10:17:26 +02:00
committed by GitHub
parent a31ab1fc37
commit f8ce186496
39 changed files with 499 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/benchmarking/src/lib.rs
+342 -12
View File
@@ -155,6 +155,12 @@ macro_rules! whitelist {
/// benchmark just like a regular benchmark, but only testing at the lowest and highest values for
/// each component. The function will return `Ok(())` if the benchmarks return no errors.
///
/// It is also possible to generate one #[test] function per benchmark by calling the
/// `impl_benchmark_test_suite` macro inside the `benchmarks` block. The functions will be named
/// `bench_<benchmark_name>` and can be run via `cargo test`.
/// You will see one line of output per benchmark. This approach will give you more understandable
/// error messages and allows for parallel benchmark execution.
///
/// You can optionally add a `verify` code block at the end of a benchmark to test any final state
/// of your benchmark in a unit test. For example:
///
@@ -174,7 +180,8 @@ macro_rules! whitelist {
///
/// These `verify` blocks will not affect your benchmark results!
///
/// You can construct benchmark tests like so:
/// You can construct benchmark by using the `impl_benchmark_test_suite` macro or
/// by manually implementing them like so:
///
/// ```ignore
/// #[test]
@@ -193,6 +200,7 @@ macro_rules! benchmarks {
$( $rest:tt )*
) => {
$crate::benchmarks_iter!(
{ }
{ }
{ }
( )
@@ -212,6 +220,7 @@ macro_rules! benchmarks_instance {
$( $rest:tt )*
) => {
$crate::benchmarks_iter!(
{ }
{ I: Instance }
{ }
( )
@@ -231,6 +240,7 @@ macro_rules! benchmarks_instance_pallet {
$( $rest:tt )*
) => {
$crate::benchmarks_iter!(
{ }
{ I: 'static }
{ }
( )
@@ -244,8 +254,60 @@ macro_rules! benchmarks_instance_pallet {
#[macro_export]
#[doc(hidden)]
macro_rules! benchmarks_iter {
// detect and extract `impl_benchmark_test_suite` call:
// - with a semi-colon
(
{ }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
impl_benchmark_test_suite!(
$bench_module:ident,
$new_test_ext:expr,
$test:path
$(, $( $args:tt )* )?);
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $bench_module, $new_test_ext, $test $(, $( $args )* )? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
$( $rest )*
}
};
// - without a semicolon
(
{ }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
impl_benchmark_test_suite!(
$bench_module:ident,
$new_test_ext:expr,
$test:path
$(, $( $args:tt )* )?)
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $bench_module, $new_test_ext, $test $(, $( $args )* )? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
$( $rest )*
}
};
// detect and extract where clause:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -255,6 +317,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound)? }
{ $( $where_bound )* }
( $( $names )* )
@@ -265,6 +328,7 @@ macro_rules! benchmarks_iter {
};
// detect and extract `#[skip_meta]` tag:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -275,6 +339,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -284,8 +349,9 @@ macro_rules! benchmarks_iter {
$( $rest )*
}
};
// detect and extract `#[extra] tag:
// detect and extract `#[extra]` tag:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -296,6 +362,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -307,6 +374,7 @@ macro_rules! benchmarks_iter {
};
// mutation arm:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* ) // This contains $( $( { $instance } )? $name:ident )*
@@ -317,6 +385,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -329,6 +398,7 @@ macro_rules! benchmarks_iter {
};
// mutation arm:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -340,6 +410,7 @@ macro_rules! benchmarks_iter {
) => {
$crate::paste::paste! {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -373,6 +444,7 @@ macro_rules! benchmarks_iter {
};
// iteration arm:
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -400,6 +472,7 @@ macro_rules! benchmarks_iter {
);
$crate::benchmarks_iter!(
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* { $( $instance )? } $name )
@@ -408,8 +481,40 @@ macro_rules! benchmarks_iter {
$( $rest )*
);
};
// iteration-exit arm
// iteration-exit arm which generates a #[test] function for each case.
(
{ $bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
) => {
$crate::selected_benchmark!(
{ $( $where_clause)* }
{ $( $instance: $instance_bound )? }
$( $names )*
);
$crate::impl_benchmark!(
{ $( $where_clause )* }
{ $( $instance: $instance_bound )? }
( $( $names )* )
( $( $names_extra ),* )
( $( $names_skip_meta ),* )
);
$crate::impl_test_function!(
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
$bench_module,
$new_test_ext,
$test
$(, $( $args )* )?
);
};
// iteration-exit arm which doesn't generate a #[test] function for all cases.
(
{ }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -431,6 +536,7 @@ macro_rules! benchmarks_iter {
};
// add verify block to _() format
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -440,6 +546,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -452,6 +559,7 @@ macro_rules! benchmarks_iter {
};
// add verify block to name() format
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -461,6 +569,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter! {
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -473,6 +582,7 @@ macro_rules! benchmarks_iter {
};
// add verify block to {} format
(
{ $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? }
{ $( $instance:ident: $instance_bound:tt )? }
{ $( $where_clause:tt )* }
( $( $names:tt )* )
@@ -482,6 +592,7 @@ macro_rules! benchmarks_iter {
$( $rest:tt )*
) => {
$crate::benchmarks_iter!(
{ $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? }
{ $( $instance: $instance_bound )? }
{ $( $where_clause )* }
( $( $names )* )
@@ -695,6 +806,100 @@ macro_rules! benchmark_backend {
};
}
// Creates #[test] functions for the given bench cases.
#[macro_export]
#[doc(hidden)]
macro_rules! impl_bench_case_tests {
(
{ $module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr }
{ $( $names_extra:tt )* }
$( { $( $bench_inst:ident )? } $bench:ident )*
)
=> {
$crate::impl_bench_name_tests!(
$module, $new_test_exec, $exec_name, $test, $extra,
{ $( $names_extra )* },
$( { $bench } )+
);
}
}
// Creates a #[test] function for the given bench name.
#[macro_export]
#[doc(hidden)]
macro_rules! impl_bench_name_tests {
// recursion anchor
(
$module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr,
{ $( $names_extra:tt )* },
{ $name:ident }
) => {
$crate::paste::paste! {
#[test]
fn [<bench_ $name>] () {
$new_test_exec.$exec_name(|| {
// Skip all #[extra] benchmarks if $extra is false.
if !($extra) {
let disabled = $crate::vec![ $( stringify!($names_extra).as_ref() ),* ];
if disabled.contains(&stringify!($name)) {
$crate::log::error!(
"INFO: extra benchmark skipped - {}",
stringify!($name),
);
return ();
}
}
// Same per-case logic as when all cases are run in the
// same function.
match std::panic::catch_unwind(|| {
$module::<$test>::[< test_benchmark_ $name >] ()
}) {
Err(err) => {
panic!("{}: {:?}", stringify!($name), err);
},
Ok(Err(err)) => {
match err {
$crate::BenchmarkError::Stop(err) => {
panic!("{}: {:?}", stringify!($name), err);
},
$crate::BenchmarkError::Override(_) => {
// This is still considered a success condition.
$crate::log::error!(
"WARNING: benchmark error overrided - {}",
stringify!($name),
);
},
$crate::BenchmarkError::Skip => {
// This is considered a success condition.
$crate::log::error!(
"WARNING: benchmark error skipped - {}",
stringify!($name),
);
}
}
},
Ok(Ok(())) => (),
}
});
}
}
};
// recursion tail
(
$module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr,
{ $( $names_extra:tt )* },
{ $name:ident } $( { $rest:ident } )+
) => {
// car
$crate::impl_bench_name_tests!($module, $new_test_exec, $exec_name, $test, $extra,
{ $( $names_extra )* }, { $name });
// cdr
$crate::impl_bench_name_tests!($module, $new_test_exec, $exec_name, $test, $extra,
{ $( $names_extra )* }, $( { $rest } )+);
};
}
// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`.
//
// Every variant must implement [`BenchmarkingSetup`].
@@ -1030,13 +1235,54 @@ macro_rules! impl_benchmark_test {
/// new_test_ext().execute_with(|| {
/// assert_ok!(test_benchmark_accumulate_dummy::<Test>());
/// assert_ok!(test_benchmark_set_dummy::<Test>());
/// assert_ok!(test_benchmark_another_set_dummy::<Test>());
/// assert_ok!(test_benchmark_sort_vector::<Test>());
/// });
/// }
/// }
/// ```
///
/// When called inside the `benchmarks` macro of the `pallet_example` as
///
/// ```rust,ignore
/// benchmarks! {
/// // Benchmarks omitted for brevity
///
/// impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test);
/// }
/// ```
///
/// It expands to the equivalent of:
///
/// ```rust,ignore
/// #[cfg(test)]
/// mod benchmarking {
/// use super::*;
/// use crate::tests::{new_test_ext, Test};
/// use frame_support::assert_ok;
///
/// #[test]
/// fn bench_accumulate_dummy() {
/// new_test_ext().execute_with(|| {
/// assert_ok!(test_benchmark_accumulate_dummy::<Test>());
/// }
/// }
///
/// #[test]
/// fn bench_set_dummy() {
/// new_test_ext().execute_with(|| {
/// assert_ok!(test_benchmark_set_dummy::<Test>());
/// }
/// }
///
/// #[test]
/// fn bench_sort_vector() {
/// new_test_ext().execute_with(|| {
/// assert_ok!(test_benchmark_sort_vector::<Test>());
/// }
/// }
/// }
/// ```
///
/// ## Arguments
///
/// The first argument, `module`, must be the path to this crate's module.
@@ -1109,16 +1355,50 @@ macro_rules! impl_benchmark_test {
// just iterate over the `Benchmarking::benchmarks` list to run the actual implementations.
#[macro_export]
macro_rules! impl_benchmark_test_suite {
// user might or might not have set some keyword arguments; set the defaults
//
// The weird syntax indicates that `rest` comes only after a comma, which is otherwise optional
(
$bench_module:ident,
$new_test_ext:expr,
$test:path
$(, $( $rest:tt )* )?
) => {
$crate::impl_benchmark_test_suite!(
$crate::impl_test_function!(
()
()
()
$bench_module,
$new_test_ext,
$test
$(, $( $rest )* )?
);
}
}
// Takes all arguments from `impl_benchmark_test_suite` and three additional arguments.
//
// Can be configured to generate one #[test] fn per bench case or
// one #[test] fn for all bench cases.
// This depends on whether or not the first argument contains a non-empty list of bench names.
#[macro_export]
#[doc(hidden)]
macro_rules! impl_test_function {
// user might or might not have set some keyword arguments; set the defaults
//
// The weird syntax indicates that `rest` comes only after a comma, which is otherwise optional
(
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
$bench_module:ident,
$new_test_ext:expr,
$test:path
$(, $( $rest:tt )* )?
) => {
$crate::impl_test_function!(
@cases:
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
@selected:
$bench_module,
$new_test_ext,
@@ -1132,6 +1412,10 @@ macro_rules! impl_benchmark_test_suite {
};
// pick off the benchmarks_path keyword argument
(
@cases:
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
@selected:
$bench_module:ident,
$new_test_ext:expr,
@@ -1143,7 +1427,11 @@ macro_rules! impl_benchmark_test_suite {
benchmarks_path = $benchmarks_path:ident
$(, $( $rest:tt )* )?
) => {
$crate::impl_benchmark_test_suite!(
$crate::impl_test_function!(
@cases:
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
@selected:
$bench_module,
$new_test_ext,
@@ -1157,6 +1445,10 @@ macro_rules! impl_benchmark_test_suite {
};
// pick off the extra keyword argument
(
@cases:
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
@selected:
$bench_module:ident,
$new_test_ext:expr,
@@ -1168,7 +1460,11 @@ macro_rules! impl_benchmark_test_suite {
extra = $extra:expr
$(, $( $rest:tt )* )?
) => {
$crate::impl_benchmark_test_suite!(
$crate::impl_test_function!(
@cases:
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
@selected:
$bench_module,
$new_test_ext,
@@ -1182,6 +1478,10 @@ macro_rules! impl_benchmark_test_suite {
};
// pick off the exec_name keyword argument
(
@cases:
( $( $names:tt )* )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
@selected:
$bench_module:ident,
$new_test_ext:expr,
@@ -1193,7 +1493,11 @@ macro_rules! impl_benchmark_test_suite {
exec_name = $exec_name:ident
$(, $( $rest:tt )* )?
) => {
$crate::impl_benchmark_test_suite!(
$crate::impl_test_function!(
@cases:
( $( $names )* )
( $( $names_extra )* )
( $( $names_skip_meta )* )
@selected:
$bench_module,
$new_test_ext,
@@ -1205,8 +1509,34 @@ macro_rules! impl_benchmark_test_suite {
$( $( $rest )* )?
);
};
// all options set; nothing else in user-provided keyword arguments
// iteration-exit arm which generates a #[test] function for each case.
(
@cases:
( $( $names:tt )+ )
( $( $names_extra:tt )* )
( $( $names_skip_meta:tt )* )
@selected:
$bench_module:ident,
$new_test_ext:expr,
$test:path,
benchmarks_path = $path_to_benchmarks_invocation:ident,
extra = $extra:expr,
exec_name = $exec_name:ident,
@user:
$(,)?
) => {
$crate::impl_bench_case_tests!(
{ $bench_module, $new_test_ext, $exec_name, $test, $extra }
{ $( $names_extra:tt )* }
$($names)+
);
};
// iteration-exit arm which generates one #[test] function for all cases.
(
@cases:
()
()
()
@selected:
$bench_module:ident,
$new_test_ext:expr,
substrate/frame/benchmarking/src/tests_instance.rs
+6 -6
View File
@@ -173,11 +173,11 @@ mod benchmarks {
} verify {
ensure!(m[0] == 0, "You forgot to sort!")
}
}
crate::impl_benchmark_test_suite!(
Pallet,
crate::tests_instance::new_test_ext(),
crate::tests_instance::Test
);
impl_benchmark_test_suite!(
Pallet,
crate::tests_instance::new_test_ext(),
crate::tests_instance::Test
)
}
}