mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-04-27 11:38:01 +00:00
Sub-commands for benchmark (#11164)
* Restructure benchmark commands Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add benchmark block test Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Fixup imports Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * CI Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Extend error message Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Apply suggestions from code review Co-authored-by: Zeke Mostov <z.mostov@gmail.com> * Review fixes Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add commands to node-template Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
This commit is contained in:
Oliver Tale-Yazdi
committed by
GitHub
GitHub
parent
ef5c4b7fc3
commit
a7261180ee
@@ -0,0 +1,493 @@
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{writer, PalletCmd};
|
||||
use codec::{Decode, Encode};
|
||||
use frame_benchmarking::{
|
||||
Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter,
|
||||
BenchmarkResult, BenchmarkSelector,
|
||||
};
|
||||
use frame_support::traits::StorageInfo;
|
||||
use linked_hash_map::LinkedHashMap;
|
||||
use sc_cli::{CliConfiguration, ExecutionStrategy, Result, SharedParams};
|
||||
use sc_client_db::BenchmarkingState;
|
||||
use sc_executor::NativeElseWasmExecutor;
|
||||
use sc_service::{Configuration, NativeExecutionDispatch};
|
||||
use sp_core::offchain::{
|
||||
testing::{TestOffchainExt, TestTransactionPoolExt},
|
||||
OffchainDbExt, OffchainWorkerExt, TransactionPoolExt,
|
||||
};
|
||||
use sp_externalities::Extensions;
|
||||
use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr};
|
||||
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
|
||||
use sp_state_machine::StateMachine;
|
||||
use std::{fmt::Debug, fs, sync::Arc, time};
|
||||
|
||||
// This takes multiple benchmark batches and combines all the results where the pallet, instance,
|
||||
// and benchmark are the same.
|
||||
fn combine_batches(
|
||||
time_batches: Vec<BenchmarkBatch>,
|
||||
db_batches: Vec<BenchmarkBatch>,
|
||||
) -> Vec<BenchmarkBatchSplitResults> {
|
||||
if time_batches.is_empty() && db_batches.is_empty() {
|
||||
return Default::default()
|
||||
}
|
||||
|
||||
let mut all_benchmarks =
|
||||
LinkedHashMap::<_, (Vec<BenchmarkResult>, Vec<BenchmarkResult>)>::new();
|
||||
|
||||
db_batches
|
||||
.into_iter()
|
||||
.for_each(|BenchmarkBatch { pallet, instance, benchmark, results }| {
|
||||
// We use this key to uniquely identify a benchmark among batches.
|
||||
let key = (pallet, instance, benchmark);
|
||||
|
||||
match all_benchmarks.get_mut(&key) {
|
||||
// We already have this benchmark, so we extend the results.
|
||||
Some(x) => x.1.extend(results),
|
||||
// New benchmark, so we add a new entry with the initial results.
|
||||
None => {
|
||||
all_benchmarks.insert(key, (Vec::new(), results));
|
||||
},
|
||||
}
|
||||
});
|
||||
|
||||
time_batches
|
||||
.into_iter()
|
||||
.for_each(|BenchmarkBatch { pallet, instance, benchmark, results }| {
|
||||
// We use this key to uniquely identify a benchmark among batches.
|
||||
let key = (pallet, instance, benchmark);
|
||||
|
||||
match all_benchmarks.get_mut(&key) {
|
||||
// We already have this benchmark, so we extend the results.
|
||||
Some(x) => x.0.extend(results),
|
||||
None => panic!("all benchmark keys should have been populated by db batches"),
|
||||
}
|
||||
});
|
||||
|
||||
all_benchmarks
|
||||
.into_iter()
|
||||
.map(|((pallet, instance, benchmark), (time_results, db_results))| {
|
||||
BenchmarkBatchSplitResults { pallet, instance, benchmark, time_results, db_results }
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
|
||||
/// Explains possible reasons why the metadata for the benchmarking could not be found.
|
||||
const ERROR_METADATA_NOT_FOUND: &'static str = "Did not find the benchmarking metadata. \
|
||||
This could mean that you either did not build the node correctly with the \
|
||||
`--features runtime-benchmarks` flag, or the chain spec that you are using was \
|
||||
not created by a node that was compiled with the flag";
|
||||
|
||||
impl PalletCmd {
|
||||
/// Runs the command and benchmarks the chain.
|
||||
pub fn run<BB, ExecDispatch>(&self, config: Configuration) -> Result<()>
|
||||
where
|
||||
BB: BlockT + Debug,
|
||||
<<<BB as BlockT>::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug,
|
||||
<BB as BlockT>::Hash: std::str::FromStr,
|
||||
ExecDispatch: NativeExecutionDispatch + 'static,
|
||||
{
|
||||
if let Some(output_path) = &self.output {
|
||||
if !output_path.is_dir() && output_path.file_name().is_none() {
|
||||
return Err("Output file or path is invalid!".into())
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(header_file) = &self.header {
|
||||
if !header_file.is_file() {
|
||||
return Err("Header file is invalid!".into())
|
||||
};
|
||||
}
|
||||
|
||||
if let Some(handlebars_template_file) = &self.template {
|
||||
if !handlebars_template_file.is_file() {
|
||||
return Err("Handlebars template file is invalid!".into())
|
||||
};
|
||||
}
|
||||
|
||||
let spec = config.chain_spec;
|
||||
let wasm_method = self.wasm_method.into();
|
||||
let strategy = self.execution.unwrap_or(ExecutionStrategy::Native);
|
||||
let pallet = self.pallet.clone().unwrap_or_else(|| String::new());
|
||||
let pallet = pallet.as_bytes();
|
||||
let extrinsic = self.extrinsic.clone().unwrap_or_else(|| String::new());
|
||||
let extrinsic_split: Vec<&str> = extrinsic.split(',').collect();
|
||||
let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect();
|
||||
|
||||
let genesis_storage = spec.build_storage()?;
|
||||
let mut changes = Default::default();
|
||||
let cache_size = Some(self.database_cache_size as usize);
|
||||
let state_with_tracking = BenchmarkingState::<BB>::new(
|
||||
genesis_storage.clone(),
|
||||
cache_size,
|
||||
self.record_proof,
|
||||
true,
|
||||
)?;
|
||||
let state_without_tracking =
|
||||
BenchmarkingState::<BB>::new(genesis_storage, cache_size, self.record_proof, false)?;
|
||||
let executor = NativeElseWasmExecutor::<ExecDispatch>::new(
|
||||
wasm_method,
|
||||
self.heap_pages,
|
||||
2, // The runtime instances cache size.
|
||||
2, // The runtime cache size
|
||||
);
|
||||
|
||||
let extensions = || -> Extensions {
|
||||
let mut extensions = Extensions::default();
|
||||
extensions.register(KeystoreExt(Arc::new(KeyStore::new()) as SyncCryptoStorePtr));
|
||||
let (offchain, _) = TestOffchainExt::new();
|
||||
let (pool, _) = TestTransactionPoolExt::new();
|
||||
extensions.register(OffchainWorkerExt::new(offchain.clone()));
|
||||
extensions.register(OffchainDbExt::new(offchain));
|
||||
extensions.register(TransactionPoolExt::new(pool));
|
||||
return extensions
|
||||
};
|
||||
|
||||
// Get Benchmark List
|
||||
let state = &state_without_tracking;
|
||||
let result = StateMachine::new(
|
||||
state,
|
||||
&mut changes,
|
||||
&executor,
|
||||
"Benchmark_benchmark_metadata",
|
||||
&(self.extra).encode(),
|
||||
extensions(),
|
||||
&sp_state_machine::backend::BackendRuntimeCode::new(state).runtime_code()?,
|
||||
sp_core::testing::TaskExecutor::new(),
|
||||
)
|
||||
.execute(strategy.into())
|
||||
.map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?;
|
||||
|
||||
let (list, storage_info) =
|
||||
<(Vec<BenchmarkList>, Vec<StorageInfo>) as Decode>::decode(&mut &result[..])
|
||||
.map_err(|e| format!("Failed to decode benchmark metadata: {:?}", e))?;
|
||||
|
||||
// Use the benchmark list and the user input to determine the set of benchmarks to run.
|
||||
let mut benchmarks_to_run = Vec::new();
|
||||
list.iter()
|
||||
.filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..])
|
||||
.for_each(|item| {
|
||||
for benchmark in &item.benchmarks {
|
||||
let benchmark_name = &benchmark.name;
|
||||
if extrinsic.is_empty() ||
|
||||
extrinsic.as_bytes() == &b"*"[..] ||
|
||||
extrinsics.contains(&&benchmark_name[..])
|
||||
{
|
||||
benchmarks_to_run.push((
|
||||
item.pallet.clone(),
|
||||
benchmark.name.clone(),
|
||||
benchmark.components.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if benchmarks_to_run.is_empty() {
|
||||
return Err("No benchmarks found which match your input.".into())
|
||||
}
|
||||
|
||||
if self.list {
|
||||
// List benchmarks instead of running them
|
||||
list_benchmark(benchmarks_to_run);
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
// Run the benchmarks
|
||||
let mut batches = Vec::new();
|
||||
let mut batches_db = Vec::new();
|
||||
let mut timer = time::SystemTime::now();
|
||||
for (pallet, extrinsic, components) in benchmarks_to_run {
|
||||
let all_components = if components.is_empty() {
|
||||
vec![Default::default()]
|
||||
} else {
|
||||
let mut all_components = Vec::new();
|
||||
for (idx, (name, low, high)) in components.iter().enumerate() {
|
||||
let lowest = self.lowest_range_values.get(idx).cloned().unwrap_or(*low);
|
||||
let highest = self.highest_range_values.get(idx).cloned().unwrap_or(*high);
|
||||
|
||||
let diff = highest - lowest;
|
||||
|
||||
// Create up to `STEPS` steps for that component between high and low.
|
||||
let step_size = (diff / self.steps).max(1);
|
||||
let num_of_steps = diff / step_size + 1;
|
||||
for s in 0..num_of_steps {
|
||||
// This is the value we will be testing for component `name`
|
||||
let component_value = lowest + step_size * s;
|
||||
|
||||
// Select the max value for all the other components.
|
||||
let c: Vec<(BenchmarkParameter, u32)> = components
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, (n, _, h))| {
|
||||
if n == name {
|
||||
(*n, component_value)
|
||||
} else {
|
||||
(*n, *self.highest_range_values.get(idx).unwrap_or(h))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
all_components.push(c);
|
||||
}
|
||||
}
|
||||
all_components
|
||||
};
|
||||
for (s, selected_components) in all_components.iter().enumerate() {
|
||||
// First we run a verification
|
||||
if !self.no_verify {
|
||||
// Dont use these results since verification code will add overhead
|
||||
let state = &state_without_tracking;
|
||||
let _results = StateMachine::new(
|
||||
state,
|
||||
&mut changes,
|
||||
&executor,
|
||||
"Benchmark_dispatch_benchmark",
|
||||
&(
|
||||
&pallet.clone(),
|
||||
&extrinsic.clone(),
|
||||
&selected_components.clone(),
|
||||
true, // run verification code
|
||||
1, // no need to do internal repeats
|
||||
)
|
||||
.encode(),
|
||||
extensions(),
|
||||
&sp_state_machine::backend::BackendRuntimeCode::new(state)
|
||||
.runtime_code()?,
|
||||
sp_core::testing::TaskExecutor::new(),
|
||||
)
|
||||
.execute(strategy.into())
|
||||
.map_err(|e| {
|
||||
format!("Error executing and verifying runtime benchmark: {}", e)
|
||||
})?;
|
||||
}
|
||||
// Do one loop of DB tracking.
|
||||
{
|
||||
let state = &state_with_tracking;
|
||||
let result = StateMachine::new(
|
||||
state, // todo remove tracking
|
||||
&mut changes,
|
||||
&executor,
|
||||
"Benchmark_dispatch_benchmark",
|
||||
&(
|
||||
&pallet.clone(),
|
||||
&extrinsic.clone(),
|
||||
&selected_components.clone(),
|
||||
false, // dont run verification code for final values
|
||||
self.repeat,
|
||||
)
|
||||
.encode(),
|
||||
extensions(),
|
||||
&sp_state_machine::backend::BackendRuntimeCode::new(state)
|
||||
.runtime_code()?,
|
||||
sp_core::testing::TaskExecutor::new(),
|
||||
)
|
||||
.execute(strategy.into())
|
||||
.map_err(|e| format!("Error executing runtime benchmark: {}", e))?;
|
||||
|
||||
let batch =
|
||||
<std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(
|
||||
&mut &result[..],
|
||||
)
|
||||
.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??;
|
||||
|
||||
batches_db.extend(batch);
|
||||
}
|
||||
// Finally run a bunch of loops to get extrinsic timing information.
|
||||
for r in 0..self.external_repeat {
|
||||
let state = &state_without_tracking;
|
||||
let result = StateMachine::new(
|
||||
state, // todo remove tracking
|
||||
&mut changes,
|
||||
&executor,
|
||||
"Benchmark_dispatch_benchmark",
|
||||
&(
|
||||
&pallet.clone(),
|
||||
&extrinsic.clone(),
|
||||
&selected_components.clone(),
|
||||
false, // dont run verification code for final values
|
||||
self.repeat,
|
||||
)
|
||||
.encode(),
|
||||
extensions(),
|
||||
&sp_state_machine::backend::BackendRuntimeCode::new(state)
|
||||
.runtime_code()?,
|
||||
sp_core::testing::TaskExecutor::new(),
|
||||
)
|
||||
.execute(strategy.into())
|
||||
.map_err(|e| format!("Error executing runtime benchmark: {}", e))?;
|
||||
|
||||
let batch =
|
||||
<std::result::Result<Vec<BenchmarkBatch>, String> as Decode>::decode(
|
||||
&mut &result[..],
|
||||
)
|
||||
.map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??;
|
||||
|
||||
batches.extend(batch);
|
||||
|
||||
// Show progress information
|
||||
if let Some(elapsed) = timer.elapsed().ok() {
|
||||
if elapsed >= time::Duration::from_secs(5) {
|
||||
timer = time::SystemTime::now();
|
||||
log::info!(
|
||||
"Running Benchmark: {}.{} {}/{} {}/{}",
|
||||
String::from_utf8(pallet.clone())
|
||||
.expect("Encoded from String; qed"),
|
||||
String::from_utf8(extrinsic.clone())
|
||||
.expect("Encoded from String; qed"),
|
||||
s + 1, // s starts at 0. todo show step
|
||||
self.steps,
|
||||
r + 1,
|
||||
self.external_repeat,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine all of the benchmark results, so that benchmarks of the same pallet/function
|
||||
// are together.
|
||||
let batches: Vec<BenchmarkBatchSplitResults> = combine_batches(batches, batches_db);
|
||||
|
||||
// Create the weights.rs file.
|
||||
if let Some(output_path) = &self.output {
|
||||
writer::write_results(&batches, &storage_info, output_path, self)?;
|
||||
}
|
||||
|
||||
// Jsonify the result and write it to a file or stdout if desired.
|
||||
if !self.jsonify(&batches)? {
|
||||
// Print the summary only if `jsonify` did not write to stdout.
|
||||
self.print_summary(&batches, &storage_info)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Jsonifies the passed batches and writes them to stdout or into a file.
|
||||
/// Can be configured via `--json` and `--json-file`.
|
||||
/// Returns whether it wrote to stdout.
|
||||
fn jsonify(&self, batches: &Vec<BenchmarkBatchSplitResults>) -> Result<bool> {
|
||||
if self.json_output || self.json_file.is_some() {
|
||||
let json = serde_json::to_string_pretty(&batches)
|
||||
.map_err(|e| format!("Serializing into JSON: {:?}", e))?;
|
||||
|
||||
if let Some(path) = &self.json_file {
|
||||
fs::write(path, json)?;
|
||||
} else {
|
||||
println!("{}", json);
|
||||
return Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Prints the results as human-readable summary without raw timing data.
|
||||
fn print_summary(
|
||||
&self,
|
||||
batches: &Vec<BenchmarkBatchSplitResults>,
|
||||
storage_info: &Vec<StorageInfo>,
|
||||
) {
|
||||
for batch in batches.into_iter() {
|
||||
// Print benchmark metadata
|
||||
println!(
|
||||
"Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}",
|
||||
String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"),
|
||||
String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"),
|
||||
self.lowest_range_values,
|
||||
self.highest_range_values,
|
||||
self.steps,
|
||||
self.repeat,
|
||||
);
|
||||
|
||||
// Skip raw data + analysis if there are no results
|
||||
if batch.time_results.is_empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
if !self.no_storage_info {
|
||||
let mut comments: Vec<String> = Default::default();
|
||||
writer::add_storage_comments(&mut comments, &batch.db_results, &storage_info);
|
||||
println!("Raw Storage Info\n========");
|
||||
for comment in comments {
|
||||
println!("{}", comment);
|
||||
}
|
||||
println!("");
|
||||
}
|
||||
|
||||
// Conduct analysis.
|
||||
if !self.no_median_slopes {
|
||||
println!("Median Slopes Analysis\n========");
|
||||
if let Some(analysis) =
|
||||
Analysis::median_slopes(&batch.time_results, BenchmarkSelector::ExtrinsicTime)
|
||||
{
|
||||
println!("-- Extrinsic Time --\n{}", analysis);
|
||||
}
|
||||
if let Some(analysis) =
|
||||
Analysis::median_slopes(&batch.db_results, BenchmarkSelector::Reads)
|
||||
{
|
||||
println!("Reads = {:?}", analysis);
|
||||
}
|
||||
if let Some(analysis) =
|
||||
Analysis::median_slopes(&batch.db_results, BenchmarkSelector::Writes)
|
||||
{
|
||||
println!("Writes = {:?}", analysis);
|
||||
}
|
||||
println!("");
|
||||
}
|
||||
if !self.no_min_squares {
|
||||
println!("Min Squares Analysis\n========");
|
||||
if let Some(analysis) =
|
||||
Analysis::min_squares_iqr(&batch.time_results, BenchmarkSelector::ExtrinsicTime)
|
||||
{
|
||||
println!("-- Extrinsic Time --\n{}", analysis);
|
||||
}
|
||||
if let Some(analysis) =
|
||||
Analysis::min_squares_iqr(&batch.db_results, BenchmarkSelector::Reads)
|
||||
{
|
||||
println!("Reads = {:?}", analysis);
|
||||
}
|
||||
if let Some(analysis) =
|
||||
Analysis::min_squares_iqr(&batch.db_results, BenchmarkSelector::Writes)
|
||||
{
|
||||
println!("Writes = {:?}", analysis);
|
||||
}
|
||||
println!("");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CliConfiguration for PalletCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
|
||||
fn chain_id(&self, _is_dev: bool) -> Result<String> {
|
||||
Ok(match self.shared_params.chain {
|
||||
Some(ref chain) => chain.clone(),
|
||||
None => "dev".into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// List the benchmarks available in the runtime, in a CSV friendly format.
|
||||
fn list_benchmark(benchmarks_to_run: Vec<(Vec<u8>, Vec<u8>, Vec<(BenchmarkParameter, u32, u32)>)>) {
|
||||
println!("pallet, benchmark");
|
||||
for (pallet, extrinsic, _components) in benchmarks_to_run {
|
||||
println!("{}, {}", String::from_utf8_lossy(&pallet), String::from_utf8_lossy(&extrinsic));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod command;
|
||||
mod writer;
|
||||
|
||||
use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
|
||||
use std::{fmt::Debug, path::PathBuf};
|
||||
|
||||
// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
|
||||
// used like crate names with `_`
|
||||
fn parse_pallet_name(pallet: &str) -> String {
|
||||
pallet.replace("-", "_")
|
||||
}
|
||||
|
||||
/// Benchmark the extrinsic weight of FRAME Pallets.
|
||||
#[derive(Debug, clap::Parser)]
|
||||
pub struct PalletCmd {
|
||||
/// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`).
|
||||
#[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")]
|
||||
pub pallet: Option<String>,
|
||||
|
||||
/// Select an extrinsic inside the pallet to benchmark, or `*` for all.
|
||||
#[clap(short, long, required_unless_present = "list")]
|
||||
pub extrinsic: Option<String>,
|
||||
|
||||
/// Select how many samples we should take across the variable components.
|
||||
#[clap(short, long, default_value = "1")]
|
||||
pub steps: u32,
|
||||
|
||||
/// Indicates lowest values for each of the component ranges.
|
||||
#[clap(long = "low", use_value_delimiter = true)]
|
||||
pub lowest_range_values: Vec<u32>,
|
||||
|
||||
/// Indicates highest values for each of the component ranges.
|
||||
#[clap(long = "high", use_value_delimiter = true)]
|
||||
pub highest_range_values: Vec<u32>,
|
||||
|
||||
/// Select how many repetitions of this benchmark should run from within the wasm.
|
||||
#[clap(short, long, default_value = "1")]
|
||||
pub repeat: u32,
|
||||
|
||||
/// Select how many repetitions of this benchmark should run from the client.
|
||||
///
|
||||
/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
|
||||
#[clap(long, default_value = "1")]
|
||||
pub external_repeat: u32,
|
||||
|
||||
/// Print the raw results in JSON format.
|
||||
#[clap(long = "json")]
|
||||
pub json_output: bool,
|
||||
|
||||
/// Write the raw results in JSON format into the given file.
|
||||
#[clap(long, conflicts_with = "json-output")]
|
||||
pub json_file: Option<PathBuf>,
|
||||
|
||||
/// Don't print the median-slopes linear regression analysis.
|
||||
#[clap(long)]
|
||||
pub no_median_slopes: bool,
|
||||
|
||||
/// Don't print the min-squares linear regression analysis.
|
||||
#[clap(long)]
|
||||
pub no_min_squares: bool,
|
||||
|
||||
/// Output the benchmarks to a Rust file at the given path.
|
||||
#[clap(long)]
|
||||
pub output: Option<PathBuf>,
|
||||
|
||||
/// Add a header file to your outputted benchmarks.
|
||||
#[clap(long)]
|
||||
pub header: Option<PathBuf>,
|
||||
|
||||
/// Path to Handlebars template file used for outputting benchmark results. (Optional)
|
||||
#[clap(long)]
|
||||
pub template: Option<PathBuf>,
|
||||
|
||||
/// Which analysis function to use when outputting benchmarks:
|
||||
/// * min-squares (default)
|
||||
/// * median-slopes
|
||||
/// * max (max of min squares and median slopes for each value)
|
||||
#[clap(long)]
|
||||
pub output_analysis: Option<String>,
|
||||
|
||||
/// Set the heap pages while running benchmarks. If not set, the default value from the client
|
||||
/// is used.
|
||||
#[clap(long)]
|
||||
pub heap_pages: Option<u64>,
|
||||
|
||||
/// Disable verification logic when running benchmarks.
|
||||
#[clap(long)]
|
||||
pub no_verify: bool,
|
||||
|
||||
/// Display and run extra benchmarks that would otherwise not be needed for weight
|
||||
/// construction.
|
||||
#[clap(long)]
|
||||
pub extra: bool,
|
||||
|
||||
/// Estimate PoV size.
|
||||
#[clap(long)]
|
||||
pub record_proof: bool,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: sc_cli::SharedParams,
|
||||
|
||||
/// The execution strategy that should be used for benchmarks.
|
||||
#[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)]
|
||||
pub execution: Option<ExecutionStrategy>,
|
||||
|
||||
/// Method for executing Wasm runtime code.
|
||||
#[clap(
|
||||
long = "wasm-execution",
|
||||
value_name = "METHOD",
|
||||
possible_values = WasmExecutionMethod::variants(),
|
||||
ignore_case = true,
|
||||
default_value = DEFAULT_WASM_EXECUTION_METHOD,
|
||||
)]
|
||||
pub wasm_method: WasmExecutionMethod,
|
||||
|
||||
/// Limit the memory the database cache can use.
|
||||
#[clap(long = "db-cache", value_name = "MiB", default_value = "1024")]
|
||||
pub database_cache_size: u32,
|
||||
|
||||
/// List the benchmarks that match your query rather than running them.
|
||||
///
|
||||
/// When nothing is provided, we list all benchmarks.
|
||||
#[clap(long)]
|
||||
pub list: bool,
|
||||
|
||||
/// If enabled, the storage info is not displayed in the output next to the analysis.
|
||||
///
|
||||
/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
|
||||
/// template for that purpose.
|
||||
#[clap(long)]
|
||||
pub no_storage_info: bool,
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{{header}}
|
||||
//! Autogenerated weights for `{{pallet}}`
|
||||
//!
|
||||
//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}}
|
||||
//! DATE: {{date}}, STEPS: `{{cmd.steps}}`, REPEAT: {{cmd.repeat}}, LOW RANGE: `{{cmd.lowest_range_values}}`, HIGH RANGE: `{{cmd.highest_range_values}}`
|
||||
//! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}}
|
||||
|
||||
// Executed Command:
|
||||
{{#each args as |arg|}}
|
||||
// {{arg}}
|
||||
{{/each}}
|
||||
|
||||
#![cfg_attr(rustfmt, rustfmt_skip)]
|
||||
#![allow(unused_parens)]
|
||||
#![allow(unused_imports)]
|
||||
|
||||
use frame_support::{traits::Get, weights::Weight};
|
||||
use sp_std::marker::PhantomData;
|
||||
|
||||
/// Weight functions for `{{pallet}}`.
|
||||
pub struct WeightInfo<T>(PhantomData<T>);
|
||||
impl<T: frame_system::Config> {{pallet}}::WeightInfo for WeightInfo<T> {
|
||||
{{#each benchmarks as |benchmark|}}
|
||||
{{#each benchmark.comments as |comment|}}
|
||||
// {{comment}}
|
||||
{{/each}}
|
||||
fn {{benchmark.name~}}
|
||||
(
|
||||
{{~#each benchmark.components as |c| ~}}
|
||||
{{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}}
|
||||
) -> Weight {
|
||||
({{underscore benchmark.base_weight}} as Weight)
|
||||
{{#each benchmark.component_weight as |cw|}}
|
||||
// Standard Error: {{underscore cw.error}}
|
||||
.saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))
|
||||
{{/each}}
|
||||
{{#if (ne benchmark.base_reads "0")}}
|
||||
.saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight))
|
||||
{{/if}}
|
||||
{{#each benchmark.component_reads as |cr|}}
|
||||
.saturating_add(T::DbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight)))
|
||||
{{/each}}
|
||||
{{#if (ne benchmark.base_writes "0")}}
|
||||
.saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight))
|
||||
{{/if}}
|
||||
{{#each benchmark.component_writes as |cw|}}
|
||||
.saturating_add(T::DbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)))
|
||||
{{/each}}
|
||||
}
|
||||
{{/each}}
|
||||
}
|
||||
@@ -0,0 +1,552 @@
|
||||
// This file is part of Substrate.
|
||||
|
||||
// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Outputs benchmark results to Rust files that can be ingested by the runtime.
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fs,
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use inflector::Inflector;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{shared::UnderscoreHelper, PalletCmd};
|
||||
use frame_benchmarking::{
|
||||
Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector,
|
||||
RegressionModel,
|
||||
};
|
||||
use frame_support::traits::StorageInfo;
|
||||
use sp_core::hexdisplay::HexDisplay;
|
||||
use sp_runtime::traits::Zero;
|
||||
|
||||
const VERSION: &'static str = env!("CARGO_PKG_VERSION");
|
||||
const TEMPLATE: &str = include_str!("./template.hbs");
|
||||
|
||||
// This is the final structure we will pass to the Handlebars template.
|
||||
#[derive(Serialize, Default, Debug, Clone)]
|
||||
struct TemplateData {
|
||||
args: Vec<String>,
|
||||
date: String,
|
||||
version: String,
|
||||
pallet: String,
|
||||
instance: String,
|
||||
header: String,
|
||||
cmd: CmdData,
|
||||
benchmarks: Vec<BenchmarkData>,
|
||||
}
|
||||
|
||||
// This was the final data we have about each benchmark.
|
||||
#[derive(Serialize, Default, Debug, Clone)]
|
||||
struct BenchmarkData {
|
||||
name: String,
|
||||
components: Vec<Component>,
|
||||
#[serde(serialize_with = "string_serialize")]
|
||||
base_weight: u128,
|
||||
#[serde(serialize_with = "string_serialize")]
|
||||
base_reads: u128,
|
||||
#[serde(serialize_with = "string_serialize")]
|
||||
base_writes: u128,
|
||||
component_weight: Vec<ComponentSlope>,
|
||||
component_reads: Vec<ComponentSlope>,
|
||||
component_writes: Vec<ComponentSlope>,
|
||||
comments: Vec<String>,
|
||||
}
|
||||
|
||||
// This forwards some specific metadata from the `PalletCmd`
|
||||
#[derive(Serialize, Default, Debug, Clone)]
|
||||
struct CmdData {
|
||||
steps: u32,
|
||||
repeat: u32,
|
||||
lowest_range_values: Vec<u32>,
|
||||
highest_range_values: Vec<u32>,
|
||||
execution: String,
|
||||
wasm_execution: String,
|
||||
chain: String,
|
||||
db_cache: u32,
|
||||
analysis_choice: String,
|
||||
}
|
||||
|
||||
// This encodes the component name and whether that component is used.
|
||||
#[derive(Serialize, Debug, Clone, Eq, PartialEq)]
|
||||
struct Component {
|
||||
name: String,
|
||||
is_used: bool,
|
||||
}
|
||||
|
||||
// This encodes the slope of some benchmark related to a component.
|
||||
#[derive(Serialize, Debug, Clone, Eq, PartialEq)]
|
||||
struct ComponentSlope {
|
||||
name: String,
|
||||
#[serde(serialize_with = "string_serialize")]
|
||||
slope: u128,
|
||||
#[serde(serialize_with = "string_serialize")]
|
||||
error: u128,
|
||||
}
|
||||
|
||||
// Small helper to create an `io::Error` from a string.
|
||||
fn io_error(s: &str) -> std::io::Error {
|
||||
use std::io::{Error, ErrorKind};
|
||||
Error::new(ErrorKind::Other, s)
|
||||
}
|
||||
|
||||
// This function takes a list of `BenchmarkBatch` and organizes them by pallet into a `HashMap`.
|
||||
// So this: `[(p1, b1), (p1, b2), (p2, b1), (p1, b3), (p2, b2)]`
|
||||
// Becomes:
|
||||
//
|
||||
// ```
|
||||
// p1 -> [b1, b2, b3]
|
||||
// p2 -> [b1, b2]
|
||||
// ```
|
||||
fn map_results(
|
||||
batches: &[BenchmarkBatchSplitResults],
|
||||
storage_info: &[StorageInfo],
|
||||
analysis_choice: &AnalysisChoice,
|
||||
) -> Result<HashMap<(String, String), Vec<BenchmarkData>>, std::io::Error> {
|
||||
// Skip if batches is empty.
|
||||
if batches.is_empty() {
|
||||
return Err(io_error("empty batches"))
|
||||
}
|
||||
|
||||
let mut all_benchmarks = HashMap::<_, Vec<BenchmarkData>>::new();
|
||||
|
||||
for batch in batches {
|
||||
// Skip if there are no results
|
||||
if batch.time_results.is_empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap();
|
||||
let instance_string = String::from_utf8(batch.instance.clone()).unwrap();
|
||||
let benchmark_data = get_benchmark_data(batch, storage_info, analysis_choice);
|
||||
let pallet_benchmarks = all_benchmarks.entry((pallet_string, instance_string)).or_default();
|
||||
pallet_benchmarks.push(benchmark_data);
|
||||
}
|
||||
Ok(all_benchmarks)
|
||||
}
|
||||
|
||||
// Get an iterator of errors from a model. If the model is `None` all errors are zero.
|
||||
fn extract_errors(model: &Option<RegressionModel>) -> impl Iterator<Item = u128> + '_ {
|
||||
let mut errors = model.as_ref().map(|m| m.se.regressor_values.iter());
|
||||
std::iter::from_fn(move || match &mut errors {
|
||||
Some(model) => model.next().map(|val| *val as u128),
|
||||
_ => Some(0),
|
||||
})
|
||||
}
|
||||
|
||||
// Analyze and return the relevant results for a given benchmark.
|
||||
fn get_benchmark_data(
|
||||
batch: &BenchmarkBatchSplitResults,
|
||||
storage_info: &[StorageInfo],
|
||||
analysis_choice: &AnalysisChoice,
|
||||
) -> BenchmarkData {
|
||||
// You can use this to put any additional comments with the benchmarking output.
|
||||
let mut comments = Vec::<String>::new();
|
||||
|
||||
// Analyze benchmarks to get the linear regression.
|
||||
let analysis_function = match analysis_choice {
|
||||
AnalysisChoice::MinSquares => Analysis::min_squares_iqr,
|
||||
AnalysisChoice::MedianSlopes => Analysis::median_slopes,
|
||||
AnalysisChoice::Max => Analysis::max,
|
||||
};
|
||||
|
||||
let extrinsic_time = analysis_function(&batch.time_results, BenchmarkSelector::ExtrinsicTime)
|
||||
.expect("analysis function should return an extrinsic time for valid inputs");
|
||||
let reads = analysis_function(&batch.db_results, BenchmarkSelector::Reads)
|
||||
.expect("analysis function should return the number of reads for valid inputs");
|
||||
let writes = analysis_function(&batch.db_results, BenchmarkSelector::Writes)
|
||||
.expect("analysis function should return the number of writes for valid inputs");
|
||||
|
||||
// Analysis data may include components that are not used, this filters out anything whose value
|
||||
// is zero.
|
||||
let mut used_components = Vec::new();
|
||||
let mut used_extrinsic_time = Vec::new();
|
||||
let mut used_reads = Vec::new();
|
||||
let mut used_writes = Vec::new();
|
||||
|
||||
extrinsic_time
|
||||
.slopes
|
||||
.into_iter()
|
||||
.zip(extrinsic_time.names.iter())
|
||||
.zip(extract_errors(&extrinsic_time.model))
|
||||
.for_each(|((slope, name), error)| {
|
||||
if !slope.is_zero() {
|
||||
if !used_components.contains(&name) {
|
||||
used_components.push(name);
|
||||
}
|
||||
used_extrinsic_time.push(ComponentSlope {
|
||||
name: name.clone(),
|
||||
slope: slope.saturating_mul(1000),
|
||||
error: error.saturating_mul(1000),
|
||||
});
|
||||
}
|
||||
});
|
||||
reads
|
||||
.slopes
|
||||
.into_iter()
|
||||
.zip(reads.names.iter())
|
||||
.zip(extract_errors(&reads.model))
|
||||
.for_each(|((slope, name), error)| {
|
||||
if !slope.is_zero() {
|
||||
if !used_components.contains(&name) {
|
||||
used_components.push(name);
|
||||
}
|
||||
used_reads.push(ComponentSlope { name: name.clone(), slope, error });
|
||||
}
|
||||
});
|
||||
writes
|
||||
.slopes
|
||||
.into_iter()
|
||||
.zip(writes.names.iter())
|
||||
.zip(extract_errors(&writes.model))
|
||||
.for_each(|((slope, name), error)| {
|
||||
if !slope.is_zero() {
|
||||
if !used_components.contains(&name) {
|
||||
used_components.push(name);
|
||||
}
|
||||
used_writes.push(ComponentSlope { name: name.clone(), slope, error });
|
||||
}
|
||||
});
|
||||
|
||||
// This puts a marker on any component which is entirely unused in the weight formula.
|
||||
let components = batch.time_results[0]
|
||||
.components
|
||||
.iter()
|
||||
.map(|(name, _)| -> Component {
|
||||
let name_string = name.to_string();
|
||||
let is_used = used_components.contains(&&name_string);
|
||||
Component { name: name_string, is_used }
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// We add additional comments showing which storage items were touched.
|
||||
add_storage_comments(&mut comments, &batch.db_results, storage_info);
|
||||
|
||||
BenchmarkData {
|
||||
name: String::from_utf8(batch.benchmark.clone()).unwrap(),
|
||||
components,
|
||||
base_weight: extrinsic_time.base.saturating_mul(1000),
|
||||
base_reads: reads.base,
|
||||
base_writes: writes.base,
|
||||
component_weight: used_extrinsic_time,
|
||||
component_reads: used_reads,
|
||||
component_writes: used_writes,
|
||||
comments,
|
||||
}
|
||||
}
|
||||
|
||||
// Create weight file from benchmark data and Handlebars template.
|
||||
pub fn write_results(
|
||||
batches: &[BenchmarkBatchSplitResults],
|
||||
storage_info: &[StorageInfo],
|
||||
path: &PathBuf,
|
||||
cmd: &PalletCmd,
|
||||
) -> Result<(), std::io::Error> {
|
||||
// Use custom template if provided.
|
||||
let template: String = match &cmd.template {
|
||||
Some(template_file) => fs::read_to_string(template_file)?,
|
||||
None => TEMPLATE.to_string(),
|
||||
};
|
||||
|
||||
// Use header if provided
|
||||
let header_text = match &cmd.header {
|
||||
Some(header_file) => {
|
||||
let text = fs::read_to_string(header_file)?;
|
||||
text
|
||||
},
|
||||
None => String::new(),
|
||||
};
|
||||
|
||||
// Date string metadata
|
||||
let date = chrono::Utc::now().format("%Y-%m-%d").to_string();
|
||||
|
||||
// Full CLI args passed to trigger the benchmark.
|
||||
let args = std::env::args().collect::<Vec<String>>();
|
||||
|
||||
// Which analysis function should be used when outputting benchmarks
|
||||
let analysis_choice: AnalysisChoice =
|
||||
cmd.output_analysis.clone().try_into().map_err(|e| io_error(e))?;
|
||||
|
||||
// Capture individual args
|
||||
let cmd_data = CmdData {
|
||||
steps: cmd.steps.clone(),
|
||||
repeat: cmd.repeat.clone(),
|
||||
lowest_range_values: cmd.lowest_range_values.clone(),
|
||||
highest_range_values: cmd.highest_range_values.clone(),
|
||||
execution: format!("{:?}", cmd.execution),
|
||||
wasm_execution: cmd.wasm_method.to_string(),
|
||||
chain: format!("{:?}", cmd.shared_params.chain),
|
||||
db_cache: cmd.database_cache_size,
|
||||
analysis_choice: format!("{:?}", analysis_choice),
|
||||
};
|
||||
|
||||
// New Handlebars instance with helpers.
|
||||
let mut handlebars = handlebars::Handlebars::new();
|
||||
handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
|
||||
handlebars.register_helper("join", Box::new(JoinHelper));
|
||||
// Don't HTML escape any characters.
|
||||
handlebars.register_escape_fn(|s| -> String { s.to_string() });
|
||||
|
||||
// Organize results by pallet into a JSON map
|
||||
let all_results = map_results(batches, storage_info, &analysis_choice)?;
|
||||
for ((pallet, instance), results) in all_results.iter() {
|
||||
let mut file_path = path.clone();
|
||||
// If a user only specified a directory...
|
||||
if file_path.is_dir() {
|
||||
// Check if there might be multiple instances benchmarked.
|
||||
if all_results.keys().any(|(p, i)| p == pallet && i != instance) {
|
||||
// Create new file: "path/to/pallet_name_instance_name.rs".
|
||||
file_path.push(pallet.clone() + "_" + &instance.to_snake_case());
|
||||
} else {
|
||||
// Create new file: "path/to/pallet_name.rs".
|
||||
file_path.push(pallet.clone());
|
||||
}
|
||||
file_path.set_extension("rs");
|
||||
}
|
||||
|
||||
let hbs_data = TemplateData {
|
||||
args: args.clone(),
|
||||
date: date.clone(),
|
||||
version: VERSION.to_string(),
|
||||
pallet: pallet.to_string(),
|
||||
instance: instance.to_string(),
|
||||
header: header_text.clone(),
|
||||
cmd: cmd_data.clone(),
|
||||
benchmarks: results.clone(),
|
||||
};
|
||||
|
||||
let mut output_file = fs::File::create(file_path)?;
|
||||
handlebars
|
||||
.render_template_to_write(&template, &hbs_data, &mut output_file)
|
||||
.map_err(|e| io_error(&e.to_string()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// This function looks at the keys touched during the benchmark, and the storage info we collected
|
||||
// from the pallets, and creates comments with information about the storage keys touched during
|
||||
// each benchmark.
|
||||
pub(crate) fn add_storage_comments(
|
||||
comments: &mut Vec<String>,
|
||||
results: &[BenchmarkResult],
|
||||
storage_info: &[StorageInfo],
|
||||
) {
|
||||
let mut storage_info_map = storage_info
|
||||
.iter()
|
||||
.map(|info| (info.prefix.clone(), info))
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
// Special hack to show `Skipped Metadata`
|
||||
let skip_storage_info = StorageInfo {
|
||||
pallet_name: b"Skipped".to_vec(),
|
||||
storage_name: b"Metadata".to_vec(),
|
||||
prefix: b"Skipped Metadata".to_vec(),
|
||||
max_values: None,
|
||||
max_size: None,
|
||||
};
|
||||
storage_info_map.insert(skip_storage_info.prefix.clone(), &skip_storage_info);
|
||||
|
||||
// Special hack to show `Benchmark Override`
|
||||
let benchmark_override = StorageInfo {
|
||||
pallet_name: b"Benchmark".to_vec(),
|
||||
storage_name: b"Override".to_vec(),
|
||||
prefix: b"Benchmark Override".to_vec(),
|
||||
max_values: None,
|
||||
max_size: None,
|
||||
};
|
||||
storage_info_map.insert(benchmark_override.prefix.clone(), &benchmark_override);
|
||||
|
||||
// This tracks the keys we already identified, so we only generate a single comment.
|
||||
let mut identified = HashSet::<Vec<u8>>::new();
|
||||
|
||||
for result in results.clone() {
|
||||
for (key, reads, writes, whitelisted) in &result.keys {
|
||||
// skip keys which are whitelisted
|
||||
if *whitelisted {
|
||||
continue
|
||||
}
|
||||
let prefix_length = key.len().min(32);
|
||||
let prefix = key[0..prefix_length].to_vec();
|
||||
if identified.contains(&prefix) {
|
||||
// skip adding comments for keys we already identified
|
||||
continue
|
||||
} else {
|
||||
// track newly identified keys
|
||||
identified.insert(prefix.clone());
|
||||
}
|
||||
match storage_info_map.get(&prefix) {
|
||||
Some(key_info) => {
|
||||
let comment = format!(
|
||||
"Storage: {} {} (r:{} w:{})",
|
||||
String::from_utf8(key_info.pallet_name.clone())
|
||||
.expect("encoded from string"),
|
||||
String::from_utf8(key_info.storage_name.clone())
|
||||
.expect("encoded from string"),
|
||||
reads,
|
||||
writes,
|
||||
);
|
||||
comments.push(comment)
|
||||
},
|
||||
None => {
|
||||
let comment = format!(
|
||||
"Storage: unknown [0x{}] (r:{} w:{})",
|
||||
HexDisplay::from(key),
|
||||
reads,
|
||||
writes,
|
||||
);
|
||||
comments.push(comment)
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A helper to join a string of vectors.
|
||||
#[derive(Clone, Copy)]
|
||||
struct JoinHelper;
|
||||
impl handlebars::HelperDef for JoinHelper {
|
||||
fn call<'reg: 'rc, 'rc>(
|
||||
&self,
|
||||
h: &handlebars::Helper,
|
||||
_: &handlebars::Handlebars,
|
||||
_: &handlebars::Context,
|
||||
_rc: &mut handlebars::RenderContext,
|
||||
out: &mut dyn handlebars::Output,
|
||||
) -> handlebars::HelperResult {
|
||||
use handlebars::JsonRender;
|
||||
let param = h.param(0).unwrap();
|
||||
let value = param.value();
|
||||
let joined = if value.is_array() {
|
||||
value
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.render())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" ")
|
||||
} else {
|
||||
value.render()
|
||||
};
|
||||
out.write(&joined)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// u128 does not serialize well into JSON for `handlebars`, so we represent it as a string.
|
||||
fn string_serialize<S>(x: &u128, s: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
s.serialize_str(&x.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use frame_benchmarking::{BenchmarkBatchSplitResults, BenchmarkParameter, BenchmarkResult};
|
||||
|
||||
fn test_data(
|
||||
pallet: &[u8],
|
||||
benchmark: &[u8],
|
||||
param: BenchmarkParameter,
|
||||
base: u32,
|
||||
slope: u32,
|
||||
) -> BenchmarkBatchSplitResults {
|
||||
let mut results = Vec::new();
|
||||
for i in 0..5 {
|
||||
results.push(BenchmarkResult {
|
||||
components: vec![(param, i), (BenchmarkParameter::z, 0)],
|
||||
extrinsic_time: (base + slope * i).into(),
|
||||
storage_root_time: (base + slope * i).into(),
|
||||
reads: (base + slope * i).into(),
|
||||
repeat_reads: 0,
|
||||
writes: (base + slope * i).into(),
|
||||
repeat_writes: 0,
|
||||
proof_size: 0,
|
||||
keys: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
return BenchmarkBatchSplitResults {
|
||||
pallet: [pallet.to_vec(), b"_pallet".to_vec()].concat(),
|
||||
instance: b"instance".to_vec(),
|
||||
benchmark: [benchmark.to_vec(), b"_benchmark".to_vec()].concat(),
|
||||
time_results: results.clone(),
|
||||
db_results: results,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_data(benchmark: &BenchmarkData, component: &str, base: u128, slope: u128) {
|
||||
assert_eq!(
|
||||
benchmark.components,
|
||||
vec![
|
||||
Component { name: component.to_string(), is_used: true },
|
||||
Component { name: "z".to_string(), is_used: false },
|
||||
],
|
||||
);
|
||||
// Weights multiplied by 1,000
|
||||
assert_eq!(benchmark.base_weight, base * 1_000);
|
||||
assert_eq!(
|
||||
benchmark.component_weight,
|
||||
vec![ComponentSlope { name: component.to_string(), slope: slope * 1_000, error: 0 }]
|
||||
);
|
||||
// DB Reads/Writes are untouched
|
||||
assert_eq!(benchmark.base_reads, base);
|
||||
assert_eq!(
|
||||
benchmark.component_reads,
|
||||
vec![ComponentSlope { name: component.to_string(), slope, error: 0 }]
|
||||
);
|
||||
assert_eq!(benchmark.base_writes, base);
|
||||
assert_eq!(
|
||||
benchmark.component_writes,
|
||||
vec![ComponentSlope { name: component.to_string(), slope, error: 0 }]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_results_works() {
|
||||
let mapped_results = map_results(
|
||||
&[
|
||||
test_data(b"first", b"first", BenchmarkParameter::a, 10, 3),
|
||||
test_data(b"first", b"second", BenchmarkParameter::b, 9, 2),
|
||||
test_data(b"second", b"first", BenchmarkParameter::c, 3, 4),
|
||||
],
|
||||
&[],
|
||||
&AnalysisChoice::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let first_benchmark = &mapped_results
|
||||
.get(&("first_pallet".to_string(), "instance".to_string()))
|
||||
.unwrap()[0];
|
||||
assert_eq!(first_benchmark.name, "first_benchmark");
|
||||
check_data(first_benchmark, "a", 10, 3);
|
||||
|
||||
let second_benchmark = &mapped_results
|
||||
.get(&("first_pallet".to_string(), "instance".to_string()))
|
||||
.unwrap()[1];
|
||||
assert_eq!(second_benchmark.name, "second_benchmark");
|
||||
check_data(second_benchmark, "b", 9, 2);
|
||||
|
||||
let second_pallet_benchmark = &mapped_results
|
||||
.get(&("second_pallet".to_string(), "instance".to_string()))
|
||||
.unwrap()[0];
|
||||
assert_eq!(second_pallet_benchmark.name, "first_benchmark");
|
||||
check_data(second_pallet_benchmark, "c", 3, 4);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user