mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-22 23:11:08 +00:00
Regression algorithm for runtime benchmarks (#5288)
* Add linregress * Regression, initial stuff. * More analytics * whitespace * Fix * Median slopes regression algo * Warnings * Update to lastest benchmark changes. * Use both algorithms temp. * Move analysis to frame. * Fix tests. * Only build analysis in std Co-authored-by: Gav Wood <gavin@parity.io>
This commit is contained in:
Marcio Diaz
Generated
+103
@@ -68,6 +68,18 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alga"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "658f9468113d34781f6ca9d014d174c74b73de870f1e0e3ad32079bbab253b19"
|
||||
dependencies = [
|
||||
"approx",
|
||||
"libm",
|
||||
"num-complex",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.11.0"
|
||||
@@ -104,6 +116,15 @@ dependencies = [
|
||||
"xdg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "approx"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "0.4.4"
|
||||
@@ -1429,6 +1450,7 @@ version = "2.0.0-alpha.4"
|
||||
dependencies = [
|
||||
"frame-support",
|
||||
"frame-system",
|
||||
"linregress",
|
||||
"parity-scale-codec",
|
||||
"sp-api",
|
||||
"sp-io",
|
||||
@@ -2595,6 +2617,12 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a"
|
||||
|
||||
[[package]]
|
||||
name = "libp2p"
|
||||
version = "0.16.2"
|
||||
@@ -3063,6 +3091,17 @@ dependencies = [
|
||||
"linked-hash-map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linregress"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9290cf6f928576eeb9c096c6fad9d8d452a0a1a70a2bbffa6e36064eedc0aac9"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"nalgebra",
|
||||
"statrs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lite-json"
|
||||
version = "0.1.0"
|
||||
@@ -3123,6 +3162,15 @@ version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4f7ec66360130972f34830bfad9ef05c6610a43938a467bcc9ab9369ab3478f"
|
||||
dependencies = [
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "maybe-uninit"
|
||||
version = "2.0.0"
|
||||
@@ -3272,6 +3320,23 @@ dependencies = [
|
||||
"unsigned-varint",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nalgebra"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aaa9fddbc34c8c35dd2108515587b8ce0cab396f17977b8c738568e4edb521a2"
|
||||
dependencies = [
|
||||
"alga",
|
||||
"approx",
|
||||
"generic-array",
|
||||
"matrixmultiply",
|
||||
"num-complex",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"rand 0.6.5",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "names"
|
||||
version = "0.11.0"
|
||||
@@ -3721,6 +3786,16 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||
dependencies = [
|
||||
"autocfg 1.0.0",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.42"
|
||||
@@ -5122,6 +5197,19 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9"
|
||||
dependencies = [
|
||||
"cloudabi",
|
||||
"fuchsia-cprng",
|
||||
"libc",
|
||||
"rand_core 0.3.1",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.6.5"
|
||||
@@ -5291,6 +5379,12 @@ dependencies = [
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rawpointer"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.3.0"
|
||||
@@ -7532,6 +7626,15 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "statrs"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8"
|
||||
dependencies = [
|
||||
"rand 0.5.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stream-cipher"
|
||||
version = "0.3.2"
|
||||
|
||||
@@ -9,6 +9,7 @@ repository = "https://github.com/paritytech/substrate/"
|
||||
description = "Macro for benchmarking a FRAME runtime."
|
||||
|
||||
[dependencies]
|
||||
linregress = "0.1"
|
||||
codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false }
|
||||
sp-api = { version = "2.0.0-alpha.4", path = "../../primitives/api", default-features = false }
|
||||
sp-runtime-interface = { version = "2.0.0-alpha.4", path = "../../primitives/runtime-interface", default-features = false }
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2020 Parity Technologies (UK) Ltd.
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Substrate is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// Substrate is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
//! Tools for analysing the benchmark results.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionModel};
|
||||
use crate::BenchmarkResults;
|
||||
|
||||
pub struct Analysis {
|
||||
base: u128,
|
||||
slopes: Vec<u128>,
|
||||
names: Vec<String>,
|
||||
value_dists: Option<Vec<(Vec<u32>, u128, u128)>>,
|
||||
model: Option<RegressionModel>,
|
||||
}
|
||||
|
||||
impl Analysis {
|
||||
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
|
||||
let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| {
|
||||
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
|
||||
for (params, _, _) in r.iter() {
|
||||
let mut p = params.iter().map(|x| x.1).collect::<Vec<_>>();
|
||||
p[i] = 0;
|
||||
*counted.entry(p).or_default() += 1;
|
||||
}
|
||||
let others: Vec<u32> = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone();
|
||||
let values = r.iter()
|
||||
.filter(|v|
|
||||
v.0.iter()
|
||||
.map(|x| x.1)
|
||||
.zip(others.iter())
|
||||
.enumerate()
|
||||
.all(|(j, (v1, v2))| j == i || v1 == *v2)
|
||||
).map(|(ps, v, _)| (ps[i].1, *v))
|
||||
.collect::<Vec<_>>();
|
||||
(format!("{:?}", param), i, others, values)
|
||||
}).collect::<Vec<_>>();
|
||||
|
||||
let models = results.iter().map(|(_, _, _, ref values)| {
|
||||
let mut slopes = vec![];
|
||||
for (i, &(x1, y1)) in values.iter().enumerate() {
|
||||
for &(x2, y2) in values.iter().skip(i + 1) {
|
||||
if x1 != x2 {
|
||||
slopes.push((y1 as f64 - y2 as f64) / (x1 as f64 - x2 as f64));
|
||||
}
|
||||
}
|
||||
}
|
||||
slopes.sort_by(|a, b| a.partial_cmp(b).expect("values well defined; qed"));
|
||||
let slope = slopes[slopes.len() / 2];
|
||||
|
||||
let mut offsets = vec![];
|
||||
for &(x, y) in values.iter() {
|
||||
offsets.push(y as f64 - slope * x as f64);
|
||||
}
|
||||
offsets.sort_by(|a, b| a.partial_cmp(b).expect("values well defined; qed"));
|
||||
let offset = offsets[offsets.len() / 2];
|
||||
|
||||
(offset, slope)
|
||||
}).collect::<Vec<_>>();
|
||||
|
||||
let models = models.iter()
|
||||
.zip(results.iter())
|
||||
.map(|((offset, slope), (_, i, others, _))| {
|
||||
let over = others.iter()
|
||||
.enumerate()
|
||||
.filter(|(j, _)| j != i)
|
||||
.map(|(j, v)| models[j].1 * *v as f64)
|
||||
.fold(0f64, |acc, i| acc + i);
|
||||
(*offset - over, *slope)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let base = models[0].0.max(0f64) as u128;
|
||||
let slopes = models.iter().map(|x| x.1.max(0f64) as u128).collect::<Vec<_>>();
|
||||
|
||||
Some(Self {
|
||||
base,
|
||||
slopes,
|
||||
names: results.into_iter().map(|x| x.0).collect::<Vec<_>>(),
|
||||
value_dists: None,
|
||||
model: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
|
||||
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
|
||||
for &(ref params, t, _) in r.iter() {
|
||||
let p = params.iter().map(|x| x.1).collect::<Vec<_>>();
|
||||
results.entry(p).or_default().push(t);
|
||||
}
|
||||
for (_, rs) in results.iter_mut() {
|
||||
rs.sort();
|
||||
let ql = rs.len() / 4;
|
||||
*rs = rs[ql..rs.len() - ql].to_vec();
|
||||
}
|
||||
|
||||
let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())];
|
||||
|
||||
let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
|
||||
data.extend(names.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| (
|
||||
p.as_str(),
|
||||
results.iter()
|
||||
.flat_map(|x| Some(x.0[i] as f64)
|
||||
.into_iter()
|
||||
.cycle()
|
||||
.take(x.1.len())
|
||||
).collect::<Vec<_>>()
|
||||
))
|
||||
);
|
||||
|
||||
let data = RegressionDataBuilder::new().build_from(data).ok()?;
|
||||
|
||||
let model = FormulaRegressionBuilder::new()
|
||||
.data(&data)
|
||||
.formula(format!("Y ~ {}", names.join(" + ")))
|
||||
.fit()
|
||||
.ok()?;
|
||||
|
||||
let slopes = model.parameters.regressor_values.iter()
|
||||
.enumerate()
|
||||
.map(|(_, x)| (*x + 0.5) as u128)
|
||||
.collect();
|
||||
|
||||
let value_dists = results.iter().map(|(p, vs)| {
|
||||
let total = vs.iter()
|
||||
.fold(0u128, |acc, v| acc + *v);
|
||||
let mean = total / vs.len() as u128;
|
||||
let sum_sq_diff = vs.iter()
|
||||
.fold(0u128, |acc, v| {
|
||||
let d = mean.max(*v) - mean.min(*v);
|
||||
acc + d * d
|
||||
});
|
||||
let stddev = (sum_sq_diff as f64 / vs.len() as f64).sqrt() as u128;
|
||||
(p.clone(), mean, stddev)
|
||||
}).collect::<Vec<_>>();
|
||||
|
||||
Some(Self {
|
||||
base: (model.parameters.intercept_value + 0.5) as u128,
|
||||
slopes,
|
||||
names,
|
||||
value_dists: Some(value_dists),
|
||||
model: Some(model),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn ms(mut nanos: u128) -> String {
|
||||
let mut x = 100_000u128;
|
||||
while x > 1 {
|
||||
if nanos > x * 1_000 {
|
||||
nanos = nanos / x * x;
|
||||
break;
|
||||
}
|
||||
x /= 10;
|
||||
}
|
||||
format!("{}", nanos as f64 / 1_000f64)
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Analysis {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
if let Some(ref value_dists) = self.value_dists {
|
||||
writeln!(f, "\nData points distribution:")?;
|
||||
writeln!(f, "{} mean µs sigma µs %", self.names.iter().map(|p| format!("{:>5}", p)).collect::<Vec<_>>().join(" "))?;
|
||||
for (param_values, mean, sigma) in value_dists.iter() {
|
||||
writeln!(f, "{} {:>8} {:>8} {:>3}.{}%",
|
||||
param_values.iter().map(|v| format!("{:>5}", v)).collect::<Vec<_>>().join(" "),
|
||||
ms(*mean),
|
||||
ms(*sigma),
|
||||
(sigma * 100 / mean),
|
||||
(sigma * 1000 / mean % 10)
|
||||
)?;
|
||||
}
|
||||
}
|
||||
if let Some(ref model) = self.model {
|
||||
writeln!(f, "\nQuality and confidence:")?;
|
||||
writeln!(f, "param error")?;
|
||||
for (p, se) in self.names.iter().zip(model.se.regressor_values.iter()) {
|
||||
writeln!(f, "{} {:>8}", p, ms(*se as u128))?;
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(f, "\nModel:")?;
|
||||
writeln!(f, "Time ~= {:>8}", ms(self.base))?;
|
||||
for (&t, n) in self.slopes.iter().zip(self.names.iter()) {
|
||||
writeln!(f, " + {} {:>8}", n, ms(t))?;
|
||||
}
|
||||
writeln!(f, " µs")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::BenchmarkParameter;
|
||||
|
||||
#[test]
|
||||
fn analysis_median_slopes_should_work() {
|
||||
let a = Analysis::median_slopes(&vec![
|
||||
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
|
||||
]).unwrap();
|
||||
assert_eq!(a.base, 10_000_000);
|
||||
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn analysis_median_min_squares_should_work() {
|
||||
let a = Analysis::min_squares_iqr(&vec![
|
||||
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
|
||||
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
|
||||
]).unwrap();
|
||||
assert_eq!(a.base, 10_000_000);
|
||||
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,12 @@
|
||||
|
||||
mod tests;
|
||||
mod utils;
|
||||
#[cfg(feature = "std")]
|
||||
mod analysis;
|
||||
|
||||
pub use utils::*;
|
||||
#[cfg(feature = "std")]
|
||||
pub use analysis::Analysis;
|
||||
#[doc(hidden)]
|
||||
pub use sp_io::storage::root as storage_root;
|
||||
pub use sp_runtime::traits::Dispatchable;
|
||||
@@ -157,7 +162,7 @@ macro_rules! benchmarks_iter {
|
||||
$( $rest:tt )*
|
||||
) => {
|
||||
$crate::benchmarks_iter! {
|
||||
{ $( $common )* } ( $( $names )* ) $name { $( $code )* }: {
|
||||
{ $( $common )* } ( $( $names )* ) $name { $( $code )* }: {
|
||||
<Call<T> as $crate::Dispatchable>::dispatch(Call::<T>::$dispatch($($arg),*), $origin.into())?;
|
||||
} $( $rest )*
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use sc_client_db::BenchmarkingState;
|
||||
use sc_service::{Configuration, ChainSpec};
|
||||
use sc_executor::{NativeExecutor, NativeExecutionDispatch};
|
||||
use codec::{Encode, Decode};
|
||||
use frame_benchmarking::BenchmarkResults;
|
||||
use frame_benchmarking::{BenchmarkResults, Analysis};
|
||||
use sp_core::{
|
||||
tasks,
|
||||
traits::KeystoreExt,
|
||||
@@ -163,6 +163,17 @@ impl BenchmarkCmd {
|
||||
print!("{:?},{:?}\n", result.1, result.2);
|
||||
});
|
||||
|
||||
print!("\n");
|
||||
|
||||
// Conduct analysis.
|
||||
if let Some(analysis) = Analysis::median_slopes(&results) {
|
||||
println!("Median Slopes Analysis\n========\n{}", analysis);
|
||||
}
|
||||
|
||||
if let Some(analysis) = Analysis::min_squares_iqr(&results) {
|
||||
println!("Min Squares Analysis\n========\n{}", analysis);
|
||||
}
|
||||
|
||||
eprintln!("Done.");
|
||||
}
|
||||
Err(error) => eprintln!("Error: {:?}", error),
|
||||
|
||||
Reference in New Issue
Block a user