pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 07:41:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support Component-less Runtime Benchmarks (#6645)

Browse Source 
* Update benchmarking macro for no components

* Handle output when error

* skip when empty

* Update analysis for zero components

* add back trace logs

* Apply suggestions from code review

* remove mean value, and use median value

* Add note

* Use standard for loop

* Apply suggestions from code review

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
This commit is contained in:
Shawn Tabrizi
2020-07-16 02:21:15 +02:00
committed by GitHub
parent e10598af1f
commit d4c21eace6
5 changed files with 385 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/benchmarking/src/analysis.rs
+30
View File
@@ -37,7 +37,35 @@ pub enum BenchmarkSelector {
}
impl Analysis {
// Useful for when there are no components, and we just need an median value of the benchmark results.
// Note: We choose the median value because it is more robust to outliers.
fn median_value(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
if r.is_empty() { return None }
let mut values: Vec<u128> = r.iter().map(|result|
match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
}
).collect();
values.sort();
let mid = values.len() / 2;
Some(Self {
base: values[mid],
slopes: Vec::new(),
names: Vec::new(),
value_dists: None,
model: None,
})
}
pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
if r[0].components.is_empty() { return Self::median_value(r, selector) }
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
for result in r.iter() {
@@ -114,6 +142,8 @@ impl Analysis {
}
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
if r[0].components.is_empty() { return Self::median_value(r, selector) }
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
for result in r.iter() {
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
substrate/frame/benchmarking/src/lib.rs
+251 -203
View File
@@ -794,100 +794,114 @@ macro_rules! impl_benchmark {
// Default number of steps for a component.
let mut prev_steps = 10;
// Select the component we will be benchmarking. Each component will be benchmarked.
for (idx, (name, low, high)) in components.iter().enumerate() {
// Get the number of steps for this component.
let steps = steps.get(idx).cloned().unwrap_or(prev_steps);
prev_steps = steps;
let repeat_benchmark = |
repeat: u32,
c: Vec<($crate::BenchmarkParameter, u32)>,
results: &mut Vec<$crate::BenchmarkResults>,
| -> Result<(), &'static str> {
// Run the benchmark `repeat` times.
for _ in 0..repeat {
// Set up the externalities environment for the setup we want to
// benchmark.
let closure_to_benchmark = <
SelectedBenchmark as $crate::BenchmarkingSetup<T>
>::instance(&selected_benchmark, &c)?;
// Skip this loop if steps is zero
if steps == 0 { continue }
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low);
let highest = highest_range_values.get(idx).cloned().unwrap_or(*high);
// Commit the externalities to the database, flushing the DB cache.
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
let diff = highest - lowest;
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
// Create up to `STEPS` steps for that component between high and low.
let step_size = (diff / steps).max(1);
let num_of_steps = diff / step_size + 1;
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?}", c
);
for s in 0..num_of_steps {
// This is the value we will be testing for component `name`
let component_value = lowest + step_size * s;
let start_extrinsic = $crate::benchmarking::current_time();
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(idx, (n, _, h))|
if n == name {
(*n, component_value)
} else {
(*n, *highest_range_values.get(idx).unwrap_or(h))
}
)
.collect();
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
// Run the benchmark `repeat` times.
for _ in 0..repeat {
// Set up the externalities environment for the setup we want to
// benchmark.
let closure_to_benchmark = <
SelectedBenchmark as $crate::BenchmarkingSetup<T>
>::instance(&selected_benchmark, &c)?;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
}
// Commit the externalities to the database, flushing the DB cache.
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
Ok(())
};
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
if components.is_empty() {
repeat_benchmark(repeat, Default::default(), &mut results)?;
} else {
// Select the component we will be benchmarking. Each component will be benchmarked.
for (idx, (name, low, high)) in components.iter().enumerate() {
// Get the number of steps for this component.
let steps = steps.get(idx).cloned().unwrap_or(prev_steps);
prev_steps = steps;
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?} {:?}", name, component_value
);
// Skip this loop if steps is zero
if steps == 0 { continue }
let start_extrinsic = $crate::benchmarking::current_time();
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low);
let highest = highest_range_values.get(idx).cloned().unwrap_or(*high);
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
let diff = highest - lowest;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Create up to `STEPS` steps for that component between high and low.
let step_size = (diff / steps).max(1);
let num_of_steps = diff / step_size + 1;
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
for s in 0..num_of_steps {
// This is the value we will be testing for component `name`
let component_value = lowest + step_size * s;
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(idx, (n, _, h))|
if n == name {
(*n, component_value)
} else {
(*n, *highest_range_values.get(idx).unwrap_or(h))
}
)
.collect();
repeat_benchmark(repeat, c, &mut results)?;
}
}
}
@@ -938,99 +952,117 @@ macro_rules! impl_benchmark {
// Default number of steps for a component.
let mut prev_steps = 10;
// Select the component we will be benchmarking. Each component will be benchmarked.
for (idx, (name, low, high)) in components.iter().enumerate() {
// Get the number of steps for this component.
let steps = steps.get(idx).cloned().unwrap_or(prev_steps);
prev_steps = steps;
let repeat_benchmark = |
repeat: u32,
c: Vec<($crate::BenchmarkParameter, u32)>,
results: &mut Vec<$crate::BenchmarkResults>,
| -> Result<(), &'static str> {
// Run the benchmark `repeat` times.
for _ in 0..repeat {
// Set up the externalities environment for the setup we want to
// benchmark.
let closure_to_benchmark = <
SelectedBenchmark as $crate::BenchmarkingSetupInstance<T, I>
>::instance(&selected_benchmark, &c)?;
// Skip this loop if steps is zero
if steps == 0 { continue }
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low);
let highest = highest_range_values.get(idx).cloned().unwrap_or(*high);
// Commit the externalities to the database, flushing the DB cache.
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
let diff = highest - lowest;
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
// Create up to `STEPS` steps for that component between high and low.
let step_size = (diff / steps).max(1);
let num_of_steps = diff / step_size + 1;
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?}",
c,
);
for s in 0..num_of_steps {
// This is the value we will be testing for component `name`
let component_value = lowest + step_size * s;
let start_extrinsic = $crate::benchmarking::current_time();
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns",
elapsed_extrinsic,
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}",
read_write_count,
);
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(idx, (n, _, h))|
if n == name {
(*n, component_value)
} else {
(*n, *highest_range_values.get(idx).unwrap_or(h))
}
)
.collect();
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
// Run the benchmark `repeat` times.
for _ in 0..repeat {
// Set up the externalities environment for the setup we want to benchmark.
let closure_to_benchmark = <
SelectedBenchmark as $crate::BenchmarkingSetupInstance<T, I>
>::instance(&selected_benchmark, &c)?;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
}
// Commit the externalities to the database, flushing the DB cache.
// This will enable worst case scenario for reading from the database.
$crate::benchmarking::commit_db();
Ok(())
};
// Reset the read/write counter so we don't count operations in the setup process.
$crate::benchmarking::reset_read_write_count();
if components.is_empty() {
repeat_benchmark(repeat, Default::default(), &mut results)?;
} else {
// Select the component we will be benchmarking. Each component will be benchmarked.
for (idx, (name, low, high)) in components.iter().enumerate() {
// Get the number of steps for this component.
let steps = steps.get(idx).cloned().unwrap_or(prev_steps);
prev_steps = steps;
// Time the extrinsic logic.
frame_support::debug::trace!(
target: "benchmark",
"Start Benchmark: {:?} {:?}", name, component_value
);
// Skip this loop if steps is zero
if steps == 0 { continue }
let start_extrinsic = $crate::benchmarking::current_time();
closure_to_benchmark()?;
let finish_extrinsic = $crate::benchmarking::current_time();
let elapsed_extrinsic = finish_extrinsic - start_extrinsic;
// Commit the changes to get proper write count
$crate::benchmarking::commit_db();
frame_support::debug::trace!(
target: "benchmark",
"End Benchmark: {} ns", elapsed_extrinsic
);
let read_write_count = $crate::benchmarking::read_write_count();
frame_support::debug::trace!(
target: "benchmark",
"Read/Write Count {:?}", read_write_count
);
let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low);
let highest = highest_range_values.get(idx).cloned().unwrap_or(*high);
// Time the storage root recalculation.
let start_storage_root = $crate::benchmarking::current_time();
$crate::storage_root();
let finish_storage_root = $crate::benchmarking::current_time();
let elapsed_storage_root = finish_storage_root - start_storage_root;
let diff = highest - lowest;
results.push($crate::BenchmarkResults {
components: c.clone(),
extrinsic_time: elapsed_extrinsic,
storage_root_time: elapsed_storage_root,
reads: read_write_count.0,
repeat_reads: read_write_count.1,
writes: read_write_count.2,
repeat_writes: read_write_count.3,
});
// Create up to `STEPS` steps for that component between high and low.
let step_size = (diff / steps).max(1);
let num_of_steps = diff / step_size + 1;
// Wipe the DB back to the genesis state.
$crate::benchmarking::wipe_db();
for s in 0..num_of_steps {
// This is the value we will be testing for component `name`
let component_value = lowest + step_size * s;
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(idx, (n, _, h))|
if n == name {
(*n, component_value)
} else {
(*n, *highest_range_values.get(idx).unwrap_or(h))
}
)
.collect();
repeat_benchmark(repeat, c, &mut results)?;
}
}
}
@@ -1060,40 +1092,48 @@ macro_rules! impl_benchmark_test {
SelectedBenchmark as $crate::BenchmarkingSetup<T>
>::components(&selected_benchmark);
assert!(
components.len() != 0,
"You need to add components to your benchmark!",
);
for (_, (name, low, high)) in components.iter().enumerate() {
// Test only the low and high value, assuming values in the middle won't break
for component_value in vec![low, high] {
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(_, (n, _, h))|
if n == name {
(*n, *component_value)
} else {
(*n, *h)
}
)
.collect();
let execute_benchmark = |
c: Vec<($crate::BenchmarkParameter, u32)>
| -> Result<(), &'static str> {
// Set up the verification state
let closure_to_verify = <
SelectedBenchmark as $crate::BenchmarkingSetup<T>
>::verify(&selected_benchmark, &c)?;
// Set up the verification state
let closure_to_verify = <
SelectedBenchmark as $crate::BenchmarkingSetup<T>
>::verify(&selected_benchmark, &c)?;
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
// Run verification
closure_to_verify()?;
// Reset the state
$crate::benchmarking::wipe_db();
Ok(())
};
if components.is_empty() {
execute_benchmark(Default::default())?;
} else {
for (_, (name, low, high)) in components.iter().enumerate() {
// Test only the low and high value, assuming values in the middle won't break
for component_value in vec![low, high] {
// Select the max value for all the other components.
let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
.enumerate()
.map(|(_, (n, _, h))|
if n == name {
(*n, *component_value)
} else {
(*n, *h)
}
)
.collect();
execute_benchmark(c)?;
}
// Run verification
closure_to_verify()?;
// Reset the state
$crate::benchmarking::wipe_db();
}
}
Ok(())
@@ -1114,6 +1154,28 @@ macro_rules! impl_benchmark_test {
SelectedBenchmark as $crate::BenchmarkingSetupInstance<T, _>
>::components(&selected_benchmark);
let execute_benchmark = |
c: Vec<($crate::BenchmarkParameter, u32)>
| -> Result<(), &'static str> {
// Set up the verification state
let closure_to_verify = <
SelectedBenchmark as $crate::BenchmarkingSetupInstance<T, _>
>::verify(&selected_benchmark, &c)?;
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Run verification
closure_to_verify()?;
// Reset the state
$crate::benchmarking::wipe_db();
Ok(())
};
for (_, (name, low, high)) in components.iter().enumerate() {
// Test only the low and high value, assuming values in the middle won't break
for component_value in vec![low, high] {
@@ -1129,21 +1191,7 @@ macro_rules! impl_benchmark_test {
)
.collect();
// Set up the verification state
let closure_to_verify = <
SelectedBenchmark as $crate::BenchmarkingSetupInstance<T, _>
>::verify(&selected_benchmark, &c)?;
// Set the block number to at least 1 so events are deposited.
if $crate::Zero::is_zero(&frame_system::Module::<T>::block_number()) {
frame_system::Module::<T>::set_block_number(1.into());
}
// Run verification
closure_to_verify()?;
// Reset the state
$crate::benchmarking::wipe_db();
execute_benchmark(c)?;
}
}
Ok(())
substrate/frame/benchmarking/src/tests.rs
+5
View File
@@ -165,6 +165,10 @@ benchmarks!{
verify {
ensure!(m[0] == 0, "You forgot to sort!")
}
no_components {
let caller = account::<T::AccountId>("caller", 0, 0);
}: set_value(RawOrigin::Signed(caller), 0)
}
#[test]
@@ -240,5 +244,6 @@ fn benchmarks_generate_unit_tests() {
assert_ok!(test_benchmark_sort_vector::<Test>());
assert_err!(test_benchmark_bad_origin::<Test>(), "Bad origin");
assert_err!(test_benchmark_bad_verify::<Test>(), "You forgot to sort!");
assert_ok!(test_benchmark_no_components::<Test>());
});
}
substrate/utils/frame/benchmarking-cli/src/command.rs
+70 -67
View File
@@ -89,78 +89,81 @@ impl BenchmarkCmd {
let results = <std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(&mut &result[..])
.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?;
if self.output {
if self.weight_trait {
let mut file = crate::writer::open_file("traits.rs")?;
crate::writer::write_trait(&mut file, results.clone())?;
} else {
let mut file = crate::writer::open_file("benchmarks.rs")?;
crate::writer::write_results(&mut file, results.clone())?;
}
}
match results {
Ok(batches) => for batch in batches.into_iter() {
// Print benchmark metadata
println!(
"Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}",
String::from_utf8(batch.pallet).expect("Encoded from String; qed"),
String::from_utf8(batch.benchmark).expect("Encoded from String; qed"),
self.lowest_range_values,
self.highest_range_values,
self.steps,
self.repeat,
);
// Skip raw data + analysis if there are no results
if batch.results.len() == 0 { continue }
if self.raw_data {
// Print the table header
batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0));
print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n");
// Print the values
batch.results.iter().for_each(|result| {
let parameters = &result.components;
parameters.iter().for_each(|param| print!("{:?},", param.1));
// Print extrinsic time and storage root time
print!("{:?},{:?},{:?},{:?},{:?},{:?}\n",
result.extrinsic_time,
result.storage_root_time,
result.reads,
result.repeat_reads,
result.writes,
result.repeat_writes,
);
});
println!();
}
// Conduct analysis.
if !self.no_median_slopes {
println!("Median Slopes Analysis\n========");
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
Ok(batches) => {
// If we are going to output results to a file...
if self.output {
if self.weight_trait {
let mut file = crate::writer::open_file("traits.rs")?;
crate::writer::write_trait(&mut file, batches.clone())?;
} else {
let mut file = crate::writer::open_file("benchmarks.rs")?;
crate::writer::write_results(&mut file, batches.clone())?;
}
}
if !self.no_min_squares {
println!("Min Squares Analysis\n========");
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
for batch in batches.into_iter() {
// Print benchmark metadata
println!(
"Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}",
String::from_utf8(batch.pallet).expect("Encoded from String; qed"),
String::from_utf8(batch.benchmark).expect("Encoded from String; qed"),
self.lowest_range_values,
self.highest_range_values,
self.steps,
self.repeat,
);
// Skip raw data + analysis if there are no results
if batch.results.is_empty() { continue }
if self.raw_data {
// Print the table header
batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0));
print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n");
// Print the values
batch.results.iter().for_each(|result| {
let parameters = &result.components;
parameters.iter().for_each(|param| print!("{:?},", param.1));
// Print extrinsic time and storage root time
print!("{:?},{:?},{:?},{:?},{:?},{:?}\n",
result.extrinsic_time,
result.storage_root_time,
result.reads,
result.repeat_reads,
result.writes,
result.repeat_writes,
);
});
println!();
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
// Conduct analysis.
if !self.no_median_slopes {
println!("Median Slopes Analysis\n========");
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
}
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
if !self.no_min_squares {
println!("Min Squares Analysis\n========");
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
}
}
}
},
substrate/utils/frame/benchmarking-cli/src/writer.rs
+29 -22
View File
@@ -30,12 +30,15 @@ pub fn open_file(path: &str) -> Result<File, std::io::Error> {
.open(path)
}
pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
let batches = batches.unwrap();
pub fn write_trait(file: &mut File, batches: Vec<BenchmarkBatch>) -> Result<(), std::io::Error> {
let mut current_pallet = Vec::<u8>::new();
batches.iter().for_each(|batch| {
// Skip writing if there are no batches
if batches.is_empty() { return Ok(()) }
for batch in &batches {
// Skip writing if there are no results
if batch.results.is_empty() { continue }
let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
@@ -55,7 +58,7 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
}
// function name
write!(file, " fn {}(", benchmark_string).unwrap();
write!(file, "\tfn {}(", benchmark_string).unwrap();
// params
let components = &batch.results[0].components;
@@ -64,7 +67,7 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
}
// return value
write!(file, ") -> Weight;\n").unwrap();
});
}
// final close trait
write!(file, "}}\n").unwrap();
@@ -72,7 +75,8 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
// Reset
current_pallet = Vec::<u8>::new();
batches.iter().for_each(|batch| {
for batch in &batches {
if batch.results.is_empty() { continue }
let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
@@ -91,7 +95,7 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
}
// function name
write!(file, " fn {}(", benchmark_string).unwrap();
write!(file, "\tfn {}(", benchmark_string).unwrap();
// params
let components = &batch.results[0].components;
@@ -100,7 +104,7 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
}
// return value
write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap();
});
}
// final close trait
write!(file, "}}\n").unwrap();
@@ -108,15 +112,18 @@ pub fn write_trait(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>
Ok(())
}
pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, String>) -> Result<(), std::io::Error> {
let batches = batches.unwrap();
pub fn write_results(file: &mut File, batches: Vec<BenchmarkBatch>) -> Result<(), std::io::Error> {
let mut current_pallet = Vec::<u8>::new();
// Skip writing if there are no batches
if batches.is_empty() { return Ok(()) }
// general imports
write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap();
batches.iter().for_each(|batch| {
for batch in &batches {
// Skip writing if there are no results
if batch.results.is_empty() { continue }
let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap();
@@ -143,7 +150,7 @@ pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, Strin
}
// function name
write!(file, " fn {}(", benchmark_string).unwrap();
write!(file, "\tfn {}(", benchmark_string).unwrap();
// params
let components = &batch.results[0].components;
@@ -154,35 +161,35 @@ pub fn write_results(file: &mut File, batches: Result<Vec<BenchmarkBatch>, Strin
write!(file, ") -> Weight {{\n").unwrap();
let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap();
write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap();
write!(file, "\t\t({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap();
extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| {
write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
write!(file, "\t\t\t.saturating_add(({} as Weight).saturating_mul({} as Weight))\n",
slope.saturating_mul(1000),
name,
).unwrap();
});
let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap();
write!(file, " .saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap();
write!(file, "\t\t\t.saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap();
reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| {
write!(file, " .saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n",
write!(file, "\t\t\t.saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n",
slope,
name,
).unwrap();
});
let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap();
write!(file, " .saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap();
write!(file, "\t\t\t.saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap();
writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| {
write!(file, " .saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n",
write!(file, "\t\t\t.saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n",
slope,
name,
).unwrap();
});
// close function
write!(file, " }}\n").unwrap();
});
write!(file, "\t}}\n").unwrap();
}
// final close trait
write!(file, "}}\n").unwrap();