pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-13 17:31:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add DB Read/Write Tracking to Benchmarking Pipeline (#6386)

Browse Source 
* initial mockup

* add and wipe

* track writes

* start to add to pipeline

* return all reads/writes

* Log reads and writes from bench db

* causes panic

* Allow multiple commits

* commit before ending benchmark

* doesn't work???

* fix

* Update lib.rs

* switch to struct for `BenchmarkResults`

* add to output

* fix test

* line width

* @kianenigma review

* Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)

* hardcoded whitelist

* Add whitelist to pipeline

* Remove whitelist pipeline from CLI, add to runtime

* clean-up unused db initialized whitelist

* Add regression analysis to DB Tracking (#6475)

* Add selector

* add tests

* debug formatter for easy formula

* Update client/db/src/bench.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

Co-authored-by: arkpar <arkady.paronyan@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
This commit is contained in:
Shawn Tabrizi
2020-06-24 21:03:55 +02:00
committed by GitHub
parent a7b31bb8d2
commit 935ee6f545
14 changed files with 471 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/benchmarking/src/analysis.rs
+105 -35
View File
@@ -29,24 +29,40 @@ pub struct Analysis {
model: Option<RegressionModel>,
}
pub enum BenchmarkSelector {
ExtrinsicTime,
StorageRootTime,
Reads,
Writes,
}
impl Analysis {
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| {
pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
for (params, _, _) in r.iter() {
let mut p = params.iter().map(|x| x.1).collect::<Vec<_>>();
for result in r.iter() {
let mut p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
p[i] = 0;
*counted.entry(p).or_default() += 1;
}
let others: Vec<u32> = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone();
let values = r.iter()
.filter(|v|
v.0.iter()
v.components.iter()
.map(|x| x.1)
.zip(others.iter())
.enumerate()
.all(|(j, (v1, v2))| j == i || v1 == *v2)
).map(|(ps, v, _)| (ps[i].1, *v))
).map(|result| {
// Extract the data we are interested in analyzing
let data = match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
};
(result.components[i].1, data)
})
.collect::<Vec<_>>();
(format!("{:?}", param), i, others, values)
}).collect::<Vec<_>>();
@@ -97,12 +113,18 @@ impl Analysis {
})
}
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
for &(ref params, t, _) in r.iter() {
let p = params.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(t);
for result in r.iter() {
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
})
}
for (_, rs) in results.iter_mut() {
rs.sort();
let ql = rs.len() / 4;
@@ -111,7 +133,7 @@ impl Analysis {
let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())];
let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
data.extend(names.iter()
.enumerate()
.map(|(i, p)| (
@@ -217,40 +239,88 @@ impl std::fmt::Display for Analysis {
}
}
impl std::fmt::Debug for Analysis {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.base)?;
for (&m, n) in self.slopes.iter().zip(self.names.iter()) {
write!(f, " + ({} * {})", m, n)?;
}
write!(f,"")
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::BenchmarkParameter;
fn benchmark_result(
components: Vec<(BenchmarkParameter, u32)>,
extrinsic_time: u128,
storage_root_time: u128,
reads: u32,
writes: u32,
) -> BenchmarkResults {
BenchmarkResults {
components,
extrinsic_time,
storage_root_time,
reads,
repeat_reads: 0,
writes,
repeat_writes: 0,
}
}
#[test]
fn analysis_median_slopes_should_work() {
let a = Analysis::median_slopes(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];
let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);
let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}
#[test]
fn analysis_median_min_squares_should_work() {
let a = Analysis::min_squares_iqr(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];
let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);
let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);
let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}
}