pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-13 22:11:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add DB Read/Write Tracking to Benchmarking Pipeline (#6386)

Browse Source 
* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* Update client/db/src/bench.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

Co-authored-by: arkpar <arkady.paronyan@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
This commit is contained in:
Shawn Tabrizi
2020-06-24 21:03:55 +02:00
committed by GitHub
parent a7b31bb8d2
commit 935ee6f545
14 changed files with 471 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/benchmarking/src/analysis.rs
+105 -35
View File
@@ -29,24 +29,40 @@ pub struct Analysis {
model: Option<RegressionModel>,
}
pub enum BenchmarkSelector {
ExtrinsicTime,
StorageRootTime,
Reads,
Writes,
}
impl Analysis {
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| {
pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
for (params, _, _) in r.iter() {
let mut p = params.iter().map(|x| x.1).collect::<Vec<_>>();
for result in r.iter() {
let mut p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
p[i] = 0;
*counted.entry(p).or_default() += 1;
}
let others: Vec<u32> = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone();
let values = r.iter()
.filter(|v|
v.0.iter()
v.components.iter()
.map(|x| x.1)
.zip(others.iter())
.enumerate()
.all(|(j, (v1, v2))| j == i || v1 == *v2)
).map(|(ps, v, _)| (ps[i].1, *v))
).map(|result| {
// Extract the data we are interested in analyzing
let data = match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
};
(result.components[i].1, data)
})
.collect::<Vec<_>>();
(format!("{:?}", param), i, others, values)
}).collect::<Vec<_>>();
@@ -97,12 +113,18 @@ impl Analysis {
})
}
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
for &(ref params, t, _) in r.iter() {
let p = params.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(t);
for result in r.iter() {
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
})
}
for (_, rs) in results.iter_mut() {
rs.sort();
let ql = rs.len() / 4;
@@ -111,7 +133,7 @@ impl Analysis {
let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())];
let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
data.extend(names.iter()
.enumerate()
.map(|(i, p)| (
@@ -217,40 +239,88 @@ impl std::fmt::Display for Analysis {
}
}
impl std::fmt::Debug for Analysis {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.base)?;
for (&m, n) in self.slopes.iter().zip(self.names.iter()) {
write!(f, " + ({} * {})", m, n)?;
}
write!(f,"")
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::BenchmarkParameter;
fn benchmark_result(
components: Vec<(BenchmarkParameter, u32)>,
extrinsic_time: u128,
storage_root_time: u128,
reads: u32,
writes: u32,
) -> BenchmarkResults {
BenchmarkResults {
components,
extrinsic_time,
storage_root_time,
reads,
repeat_reads: 0,
writes,
repeat_writes: 0,
}
}
#[test]
fn analysis_median_slopes_should_work() {
let a = Analysis::median_slopes(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];
let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);
let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}
#[test]
fn analysis_median_min_squares_should_work() {
let a = Analysis::min_squares_iqr(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];
let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);
let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}
}
substrate/frame/benchmarking/src/lib.rs
+50 -6
View File
@@ -26,7 +26,7 @@ mod analysis;
pub use utils::*;
#[cfg(feature = "std")]
pub use analysis::Analysis;
pub use analysis::{Analysis, BenchmarkSelector};
#[doc(hidden)]
pub use sp_io::storage::root as storage_root;
pub use sp_runtime::traits::Zero;
@@ -771,6 +771,7 @@ macro_rules! impl_benchmark {
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
whitelist: &[Vec<u8>]
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -780,6 +781,9 @@ macro_rules! impl_benchmark {
_ => return Err("Could not find extrinsic."),
};
// Add whitelist to DB
$crate::benchmarking::set_whitelist(whitelist.to_vec());
// Warm up the DB
$crate::benchmarking::commit_db();
$crate::benchmarking::wipe_db();
@@ -841,6 +845,9 @@ macro_rules! impl_benchmark {
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
@@ -851,11 +858,17 @@ macro_rules! impl_benchmark {
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
@@ -863,7 +876,15 @@ macro_rules! impl_benchmark {
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root));
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
@@ -892,6 +913,7 @@ macro_rules! impl_benchmark {
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
whitelist: &[Vec<u8>]
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic)
@@ -901,6 +923,9 @@ macro_rules! impl_benchmark {
_ => return Err("Could not find extrinsic."),
};
// Add whitelist to DB
$crate::benchmarking::set_whitelist(whitelist.to_vec());
// Warm up the DB
$crate::benchmarking::commit_db();
$crate::benchmarking::wipe_db();
@@ -963,6 +988,9 @@ macro_rules! impl_benchmark {
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
@@ -973,11 +1001,17 @@ macro_rules! impl_benchmark {
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
@@ -985,7 +1019,15 @@ macro_rules! impl_benchmark {
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root));
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
@@ -1139,7 +1181,7 @@ macro_rules! impl_benchmark_test {
#[macro_export]
macro_rules! add_benchmark {
( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params;
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params;
if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
for benchmark in $( $location )*::benchmarks().into_iter() {
@@ -1150,6 +1192,7 @@ macro_rules! add_benchmark {
&highest_range_values[..],
&steps[..],
repeat,
whitelist,
)?,
pallet: $name.to_vec(),
benchmark: benchmark.to_vec(),
@@ -1163,6 +1206,7 @@ macro_rules! add_benchmark {
&highest_range_values[..],
&steps[..],
repeat,
whitelist,
)?,
pallet: $name.to_vec(),
benchmark: benchmark.clone(),
substrate/frame/benchmarking/src/utils.rs
+25 -1
View File
@@ -44,7 +44,16 @@ pub struct BenchmarkBatch {
/// Results from running benchmarks on a FRAME pallet.
/// Contains duration of the function call in nanoseconds along with the benchmark parameters
/// used for that benchmark result.
pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128, u128);
#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
pub struct BenchmarkResults {
pub components: Vec<(BenchmarkParameter, u32)>,
pub extrinsic_time: u128,
pub storage_root_time: u128,
pub reads: u32,
pub repeat_reads: u32,
pub writes: u32,
pub repeat_writes: u32,
}
sp_api::decl_runtime_apis! {
/// Runtime api for benchmarking a FRAME runtime.
@@ -83,6 +92,20 @@ pub trait Benchmarking {
fn commit_db(&mut self) {
self.commit()
}
/// Get the read/write count
fn read_write_count(&self) -> (u32, u32, u32, u32) {
self.read_write_count()
}
/// Reset the read/write count
fn reset_read_write_count(&mut self) {
self.reset_read_write_count()
}
fn set_whitelist(&mut self, new: Vec<Vec<u8>>) {
self.set_whitelist(new)
}
}
/// The pallet benchmarking trait.
@@ -106,6 +129,7 @@ pub trait Benchmarking<T> {
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
whitelist: &[Vec<u8>]
) -> Result<Vec<T>, &'static str>;
}