diff --git a/substrate/.maintain/frame-weight-template.hbs b/substrate/.maintain/frame-weight-template.hbs new file mode 100644 index 0000000000..d0d7ef93d3 --- /dev/null +++ b/substrate/.maintain/frame-weight-template.hbs @@ -0,0 +1,68 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Weights for {{pallet}} +//! {{join args}} +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}}, STEPS: {{cmd.steps}}, REPEAT: {{cmd.repeat}}, LOW RANGE: {{cmd.lowest_range_values}}, HIGH RANGE: {{cmd.highest_range_values}} +//! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}} + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::Weight}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for {{pallet}}. +pub trait WeightInfo { + {{#each benchmarks as |benchmark| ~}} + fn {{benchmark.name~}} + ( + {{~#each benchmark.components as |c| ~}} + _{{c.name}}: u32, {{/each~}} + ) -> Weight; + {{/each}} +} + +/// Weights for {{pallet}} using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + {{#each benchmarks as |benchmark| ~}} + fn {{benchmark.name~}} + ( + {{~#each benchmark.components as |c| ~}} + {{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}} + ) -> Weight { + ({{underscore benchmark.base_weight}} as Weight) + {{#each benchmark.component_weight as |cw| ~}} + .saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)) + {{/each}} + {{~#if (ne benchmark.base_reads "0") ~}} + .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) + {{/if}} + {{~#each benchmark.component_reads as |cr| ~}} + .saturating_add(T::DbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight))) + {{/each}} + {{~#if (ne benchmark.base_writes "0") ~}} + .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) + {{/if}} + {{~#each benchmark.component_writes as |cw| ~}} + .saturating_add(T::DbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))) + {{~/each}} + } + {{/each}} +} diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 1560b7bddd..f26f4b2ae0 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -1231,7 +1231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4d33be9473d06f75f58220f71f7a9317aca647dc061dbd3c361b0bef505fbea" dependencies = [ "byteorder 1.3.4", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -1599,11 +1599,13 @@ version = "2.0.0" dependencies = [ "chrono", "frame-benchmarking", + "handlebars", "parity-scale-codec", "sc-cli", "sc-client-db", "sc-executor", "sc-service", + "serde", "sp-core", "sp-externalities", "sp-keystore", @@ -2181,6 +2183,20 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" +[[package]] +name = "handlebars" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcd1b5399b9884f9ae18b5d4105d180720c8f602aeb73d3ceae9d6b1d13a5fa7" +dependencies = [ + "log", + "pest", + "pest_derive", + "quick-error 2.0.0", + "serde", + "serde_json", +] + [[package]] name = "hash-db" version = "0.15.2" @@ -2350,7 +2366,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -3407,6 +3423,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.0.1" @@ -5441,6 +5463,49 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + [[package]] name = "petgraph" version = "0.5.1" @@ -5734,6 +5799,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda" + [[package]] name = "quickcheck" version = "0.9.2" @@ -7653,9 +7724,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" [[package]] name = "serde" -version = "1.0.114" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +checksum = "96fe57af81d28386a513cbc6858332abc6117cfdb5999647c6444b8f43a370a5" dependencies = [ "serde_derive", ] @@ -7672,9 +7743,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.114" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +checksum = "f630a6370fd8e457873b4bd2ffdae75408bc291ba72be773772a4c2a065d9ae8" dependencies = [ "proc-macro2", "quote", @@ -9700,6 +9771,12 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "uint" version = "0.8.3" diff --git a/substrate/frame/benchmarking/README.md b/substrate/frame/benchmarking/README.md index bf4bf951aa..1727072709 100644 --- a/substrate/frame/benchmarking/README.md +++ b/substrate/frame/benchmarking/README.md @@ -116,10 +116,15 @@ need to move into your node's binary folder. For example, with the Substrate rep you would test the Balances pallet's benchmarks: ```bash -cd bin/node/cli cargo test -p pallet-balances --features runtime-benchmarks ``` +> NOTE: Substrate uses a virtual workspace which does not allow you to compile with feature flags. +> ``` +> error: --features is not allowed in the root of a virtual workspace` +> ``` +> To solve this, navigate to the folder of the node (`cd bin/node/cli`) or pallet (`cd frame/pallet`) and run the command there. + ## Adding Benchmarks The benchmarks included with each pallet are not automatically added to your node. To actually @@ -163,14 +168,14 @@ Then you can run a benchmark like so: ```bash ./target/release/substrate benchmark \ - --chain dev \ # Configurable Chain Spec - --execution=wasm \ # Always test with Wasm - --wasm-execution=compiled \ # Always used `wasm-time` - --pallet pallet_balances \ # Select the pallet - --extrinsic transfer \ # Select the extrinsic - --steps 50 \ # Number of samples across component ranges - --repeat 20 \ # Number of times we repeat a benchmark - --output \ # Output benchmark results into a Rust file + --chain dev \ # Configurable Chain Spec + --execution=wasm \ # Always test with Wasm + --wasm-execution=compiled \ # Always used `wasm-time` + --pallet pallet_balances \ # Select the pallet + --extrinsic transfer \ # Select the extrinsic + --steps 50 \ # Number of samples across component ranges + --repeat 20 \ # Number of times we repeat a benchmark + --output \ # Output benchmark results into a folder or file ``` This will output a file `pallet_name.rs` which implements the `WeightInfo` trait you should include @@ -179,6 +184,19 @@ implementation of the `WeightInfo` trait. This means that you will be able to us Substrate pallets while still keeping your network safe for your specific configuration and requirements. +The benchmarking CLI uses a Handlebars template to format the final output file. You can optionally +pass the flag `--template` pointing to a custom template that can be used instead. Within the +template, you have access to all the data provided by the `TemplateData` struct in the +[benchmarking CLI writer](../../utils/frame/benchmarking-cli/src/writer.rs). You can find the +default template used [here](../../utils/frame/benchmarking-cli/src/template.hbs). + +There are some custom Handlebars helpers included with our output generation: + +* `underscore`: Add an underscore to every 3rd character from the right of a string. Primarily to be +used for delimiting large numbers. +* `join`: Join an array of strings into a space-separated string for the template. Primarily to be +used for joining all the arguments passed to the CLI. + To get a full list of available options when running benchmarks, run: ```bash diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index f2c227f782..4ee2454e70 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -26,6 +26,8 @@ sp-state-machine = { version = "0.8.0", path = "../../../primitives/state-machin structopt = "0.3.8" codec = { version = "1.3.1", package = "parity-scale-codec" } chrono = "0.4" +serde = "1.0.116" +handlebars = "3.5.0" [features] default = ["db"] diff --git a/substrate/utils/frame/benchmarking-cli/src/command.rs b/substrate/utils/frame/benchmarking-cli/src/command.rs index f5ea83d7b0..00a2e7bd7f 100644 --- a/substrate/utils/frame/benchmarking-cli/src/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/command.rs @@ -43,13 +43,19 @@ impl BenchmarkCmd { ExecDispatch: NativeExecutionDispatch + 'static, { if let Some(output_path) = &self.output { - if !output_path.is_dir() { return Err("Output path is invalid!".into()) }; + if !output_path.is_dir() && output_path.file_name().is_none() { + return Err("Output file or path is invalid!".into()) + } } if let Some(header_file) = &self.header { if !header_file.is_file() { return Err("Header file is invalid!".into()) }; } + if let Some(handlebars_template_file) = &self.template { + if !handlebars_template_file.is_file() { return Err("Handlebars template file is invalid!".into()) }; + } + let spec = config.chain_spec; let wasm_method = self.wasm_method.into(); let strategy = self.execution.unwrap_or(ExecutionStrategy::Native); @@ -99,13 +105,8 @@ impl BenchmarkCmd { match results { Ok(batches) => { - // If we are going to output results to a file... if let Some(output_path) = &self.output { - if self.trait_def { - crate::writer::write_trait(&batches, output_path, self)?; - } else { - crate::writer::write_results(&batches, output_path, self)?; - } + crate::writer::write_results(&batches, output_path, self)?; } for batch in batches.into_iter() { diff --git a/substrate/utils/frame/benchmarking-cli/src/lib.rs b/substrate/utils/frame/benchmarking-cli/src/lib.rs index 725ed3113b..b89bceeb95 100644 --- a/substrate/utils/frame/benchmarking-cli/src/lib.rs +++ b/substrate/utils/frame/benchmarking-cli/src/lib.rs @@ -68,9 +68,9 @@ pub struct BenchmarkCmd { #[structopt(long)] pub header: Option, - /// Output the trait definition to a Rust file. + /// Path to Handlebars template file used for outputting benchmark results. (Optional) #[structopt(long)] - pub trait_def: bool, + pub template: Option, /// Set the heap pages while running benchmarks. #[structopt(long)] @@ -84,18 +84,6 @@ pub struct BenchmarkCmd { #[structopt(long)] pub extra: bool, - /// Output files using spaces instead of tabs. - #[structopt(long)] - pub spaces: bool, - - /// Output benchmarks file using this struct name. - #[structopt(long, default_value = "WeightInfo")] - pub r#struct: String, - - /// Output benchmarks file using this trait name. - #[structopt(long, default_value = "WeightInfo")] - pub r#trait: String, - #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, diff --git a/substrate/utils/frame/benchmarking-cli/src/template.hbs b/substrate/utils/frame/benchmarking-cli/src/template.hbs new file mode 100644 index 0000000000..3a7e57c954 --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/template.hbs @@ -0,0 +1,41 @@ +{{header}} +//! Weights for {{pallet}} +//! {{join args}} +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}}, STEPS: {{cmd.steps}}, REPEAT: {{cmd.repeat}}, LOW RANGE: {{cmd.lowest_range_values}}, HIGH RANGE: {{cmd.highest_range_values}} +//! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}} + +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::Weight}; +use sp_std::marker::PhantomData; + +/// Weight functions for {{pallet}}. +pub struct WeightInfo(PhantomData); +impl {{pallet}}::WeightInfo for WeightInfo { + {{#each benchmarks as |benchmark| ~}} + fn {{benchmark.name~}} + ( + {{~#each benchmark.components as |c| ~}} + {{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}} + ) -> Weight { + ({{underscore benchmark.base_weight}} as Weight) + {{#each benchmark.component_weight as |cw| ~}} + .saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)) + {{/each}} + {{~#if (ne benchmark.base_reads "0") ~}} + .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) + {{/if}} + {{~#each benchmark.component_reads as |cr| ~}} + .saturating_add(T::DbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight))) + {{/each}} + {{~#if (ne benchmark.base_writes "0") ~}} + .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) + {{/if}} + {{~#each benchmark.component_writes as |cw| ~}} + .saturating_add(T::DbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))) + {{~/each}} + } + {{/each}} +} diff --git a/substrate/utils/frame/benchmarking-cli/src/writer.rs b/substrate/utils/frame/benchmarking-cli/src/writer.rs index 23c1db06fb..6142300023 100644 --- a/substrate/utils/frame/benchmarking-cli/src/writer.rs +++ b/substrate/utils/frame/benchmarking-cli/src/writer.rs @@ -17,23 +17,257 @@ // Outputs benchmark results to Rust files that can be ingested by the runtime. -use crate::BenchmarkCmd; -use std::fs::{self, File, OpenOptions}; -use std::io::prelude::*; +use std::collections::HashMap; +use std::fs; use std::path::PathBuf; + +use serde::Serialize; + +use crate::BenchmarkCmd; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; use sp_runtime::traits::Zero; const VERSION: &'static str = env!("CARGO_PKG_VERSION"); +const TEMPLATE: &str = include_str!("./template.hbs"); -pub fn open_file(path: PathBuf) -> Result { - OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(path) +// This is the final structure we will pass to the Handlebars template. +#[derive(Serialize, Default, Debug, Clone)] +struct TemplateData { + args: Vec, + date: String, + version: String, + pallet: String, + header: String, + cmd: CmdData, + benchmarks: Vec, } +// This was the final data we have about each benchmark. +#[derive(Serialize, Default, Debug, Clone)] +struct BenchmarkData { + name: String, + components: Vec, + #[serde(serialize_with = "string_serialize")] + base_weight: u128, + #[serde(serialize_with = "string_serialize")] + base_reads: u128, + #[serde(serialize_with = "string_serialize")] + base_writes: u128, + component_weight: Vec, + component_reads: Vec, + component_writes: Vec, +} + +// This forwards some specific metadata from the `BenchmarkCmd` +#[derive(Serialize, Default, Debug, Clone)] +struct CmdData { + steps: Vec, + repeat: u32, + lowest_range_values: Vec, + highest_range_values: Vec, + execution: String, + wasm_execution: String, + chain: String, + db_cache: u32, +} + +// This encodes the component name and whether that component is used. +#[derive(Serialize, Debug, Clone, Eq, PartialEq)] +struct Component { + name: String, + is_used: bool, +} + +// This encodes the slope of some benchmark related to a component. +#[derive(Serialize, Debug, Clone, Eq, PartialEq)] +struct ComponentSlope { + name: String, + #[serde(serialize_with = "string_serialize")] + slope: u128, +} + +// Small helper to create an `io::Error` from a string. +fn io_error(s: &str) -> std::io::Error { + use std::io::{Error, ErrorKind}; + Error::new(ErrorKind::Other, s) +} + +// This function takes a list of `BenchmarkBatch` and organizes them by pallet into a `HashMap`. +// So this: `[(p1, b1), (p1, b2), (p1, b3), (p2, b1), (p2, b2)]` +// Becomes: +// +// ``` +// p1 -> [b1, b2, b3] +// p2 -> [b1, b2] +// ``` +fn map_results(batches: &[BenchmarkBatch]) -> Result>, std::io::Error> { + // Skip if batches is empty. + if batches.is_empty() { return Err(io_error("empty batches")) } + + let mut all_benchmarks = HashMap::new(); + let mut pallet_benchmarks = Vec::new(); + + let mut batches_iter = batches.iter().peekable(); + while let Some(batch) = batches_iter.next() { + // Skip if there are no results + if batch.results.is_empty() { continue } + + let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark_data = get_benchmark_data(batch); + pallet_benchmarks.push(benchmark_data); + + // Check if this is the end of the iterator + if let Some(next) = batches_iter.peek() { + // Next pallet is different than current pallet, save and create new data. + let next_pallet = String::from_utf8(next.pallet.clone()).unwrap(); + if next_pallet != pallet_string { + all_benchmarks.insert(pallet_string, pallet_benchmarks.clone()); + pallet_benchmarks = Vec::new(); + } + } else { + // This is the end of the iterator, so push the final data. + all_benchmarks.insert(pallet_string, pallet_benchmarks.clone()); + } + } + Ok(all_benchmarks) +} + +// Analyze and return the relevant results for a given benchmark. +fn get_benchmark_data(batch: &BenchmarkBatch) -> BenchmarkData { + // Analyze benchmarks to get the linear regression. + let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); + let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); + let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); + + // Analysis data may include components that are not used, this filters out anything whose value is zero. + let mut used_components = Vec::new(); + let mut used_extrinsic_time = Vec::new(); + let mut used_reads = Vec::new(); + let mut used_writes = Vec::new(); + + extrinsic_time.slopes.into_iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_extrinsic_time.push(ComponentSlope { + name: name.clone(), + slope: slope.saturating_mul(1000), + }); + } + }); + reads.slopes.into_iter().zip(reads.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_reads.push(ComponentSlope { name: name.clone(), slope }); + } + }); + writes.slopes.into_iter().zip(writes.names.iter()).for_each(|(slope, name)| { + if !slope.is_zero() { + if !used_components.contains(&name) { used_components.push(name); } + used_writes.push(ComponentSlope { name: name.clone(), slope }); + } + }); + + // This puts a marker on any component which is entirely unused in the weight formula. + let components = batch.results[0].components + .iter() + .map(|(name, _)| -> Component { + let name_string = name.to_string(); + let is_used = used_components.contains(&&name_string); + Component { name: name_string, is_used } + }) + .collect::>(); + + BenchmarkData { + name: String::from_utf8(batch.benchmark.clone()).unwrap(), + components, + base_weight: extrinsic_time.base.saturating_mul(1000), + base_reads: reads.base, + base_writes: writes.base, + component_weight: used_extrinsic_time, + component_reads: used_reads, + component_writes: used_writes, + } +} + +// Create weight file from benchmark data and Handlebars template. +pub fn write_results( + batches: &[BenchmarkBatch], + path: &PathBuf, + cmd: &BenchmarkCmd, +) -> Result<(), std::io::Error> { + // Use custom template if provided. + let template: String = match &cmd.template { + Some(template_file) => { + fs::read_to_string(template_file)? + }, + None => { + TEMPLATE.to_string() + }, + }; + + // Use header if provided + let header_text = match &cmd.header { + Some(header_file) => { + let text = fs::read_to_string(header_file)?; + text + }, + None => String::new(), + }; + + // Date string metadata + let date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + + // Full CLI args passed to trigger the benchmark. + let args = std::env::args().collect::>(); + + // Capture individual args + let cmd_data = CmdData { + steps: cmd.steps.clone(), + repeat: cmd.repeat.clone(), + lowest_range_values: cmd.lowest_range_values.clone(), + highest_range_values: cmd.highest_range_values.clone(), + execution: format!("{:?}", cmd.execution), + wasm_execution: cmd.wasm_method.to_string(), + chain: format!("{:?}", cmd.shared_params.chain), + db_cache: cmd.database_cache_size, + }; + + // New Handlebars instance with helpers. + let mut handlebars = handlebars::Handlebars::new(); + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); + handlebars.register_helper("join", Box::new(JoinHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + + // Organize results by pallet into a JSON map + let all_results = map_results(batches)?; + for (pallet, results) in all_results.into_iter() { + let mut file_path = path.clone(); + // If a user only specified a directory... + if file_path.is_dir() { + // Create new file: "path/to/pallet_name.rs". + file_path.push(&pallet); + file_path.set_extension("rs"); + } + + let hbs_data = TemplateData { + args: args.clone(), + date: date.clone(), + version: VERSION.to_string(), + pallet: pallet, + header: header_text.clone(), + cmd: cmd_data.clone(), + benchmarks: results, + }; + + let mut output_file = fs::File::create(file_path)?; + handlebars.render_template_to_write(&template, &hbs_data, &mut output_file) + .map_err(|e| io_error(&e.to_string()))?; + } + Ok(()) +} + +// Add an underscore after every 3rd character, i.e. a separator for large numbers. fn underscore(i: Number) -> String where Number: std::string::ToString { @@ -49,282 +283,135 @@ fn underscore(i: Number) -> String s } -pub fn write_trait( - batches: &[BenchmarkBatch], - path: &PathBuf, - cmd: &BenchmarkCmd, -) -> Result<(), std::io::Error> { - let mut file_path = path.clone(); - file_path.push("trait"); - file_path.set_extension("rs"); - let mut file = crate::writer::open_file(file_path)?; - - let indent = if cmd.spaces {" "} else {"\t"}; - - let mut current_pallet = Vec::::new(); - - // Skip writing if there are no batches - if batches.is_empty() { return Ok(()) } - - for batch in batches { - // Skip writing if there are no results - if batch.results.is_empty() { continue } - - let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); - let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); - - // only create new trait definitions when we go to a new pallet - if batch.pallet != current_pallet { - if !current_pallet.is_empty() { - // close trait - write!(file, "}}\n")?; - } - - // trait wrapper - write!(file, "// {}\n", pallet_string)?; - write!(file, "pub trait {} {{\n", cmd.r#trait)?; - - current_pallet = batch.pallet.clone() - } - - // function name - write!(file, "{}fn {}(", indent, benchmark_string)?; - - // params - let components = &batch.results[0].components; - for component in components { - write!(file, "{:?}: u32, ", component.0)?; - } - // return value - write!(file, ") -> Weight;\n")?; +// A Handlebars helper to add an underscore after every 3rd character, +// i.e. a separator for large numbers. +#[derive(Clone, Copy)] +struct UnderscoreHelper; +impl handlebars::HelperDef for UnderscoreHelper { + fn call<'reg: 'rc, 'rc>( + &self, h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).unwrap(); + let underscore_param = underscore(param.value().render()); + out.write(&underscore_param)?; + Ok(()) } - - // final close trait - write!(file, "}}\n")?; - - Ok(()) } -pub fn write_results( - batches: &[BenchmarkBatch], - path: &PathBuf, - cmd: &BenchmarkCmd, -) -> Result<(), std::io::Error> { - - let header_text = match &cmd.header { - Some(header_file) => { - let text = fs::read_to_string(header_file)?; - Some(text) - }, - None => None, - }; - - let indent = if cmd.spaces {" "} else {"\t"}; - let date = chrono::Utc::now(); - - let mut current_pallet = Vec::::new(); - - // Skip writing if there are no batches - if batches.is_empty() { return Ok(()) } - - let mut batches_iter = batches.iter().peekable(); - - let first_pallet = String::from_utf8( - batches_iter.peek().expect("we checked that batches is not empty").pallet.clone() - ).unwrap(); - - let mut file_path = path.clone(); - file_path.push(first_pallet); - file_path.set_extension("rs"); - - let mut file = open_file(file_path)?; - - while let Some(batch) = batches_iter.next() { - // Skip writing if there are no results - if batch.results.is_empty() { continue } - - let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); - let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); - - // only create new trait definitions when we go to a new pallet - if batch.pallet != current_pallet { - // optional header and copyright - if let Some(header) = &header_text { - write!(file, "{}\n", header)?; - } - - // title of file - write!(file, "//! Weights for {}\n", pallet_string)?; - - // auto-generation note - write!( - file, - "//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {}\n", - VERSION, - )?; - - // date of generation + some settings - write!( - file, - "//! DATE: {}, STEPS: {:?}, REPEAT: {}, LOW RANGE: {:?}, HIGH RANGE: {:?}\n", - date.format("%Y-%m-%d"), - cmd.steps, - cmd.repeat, - cmd.lowest_range_values, - cmd.highest_range_values, - )?; - - // more settings - write!( - file, - "//! EXECUTION: {:?}, WASM-EXECUTION: {}, CHAIN: {:?}, DB CACHE: {}\n", - cmd.execution, - cmd.wasm_method, - cmd.shared_params.chain, - cmd.database_cache_size, - )?; - - // allow statements - write!( - file, - "#![allow(unused_parens)]\n#![allow(unused_imports)]\n\n", - )?; - - // general imports - write!( - file, - "use frame_support::{{traits::Get, weights::Weight}};\nuse sp_std::marker::PhantomData;\n\n" - )?; - - // struct for weights - write!(file, "pub struct {}(PhantomData);\n", cmd.r#struct)?; - - // trait wrapper - write!( - file, - "impl {}::{} for {} {{\n", - pallet_string, - cmd.r#trait, - cmd.r#struct, - )?; - - current_pallet = batch.pallet.clone() - } - - // Analysis results - let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - - // Analysis data may include components that are not used, this filters out anything whose value is zero. - let mut used_components = Vec::new(); - let mut used_extrinsic_time = Vec::new(); - let mut used_reads = Vec::new(); - let mut used_writes = Vec::new(); - extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { - if !slope.is_zero() { - if !used_components.contains(&name) { used_components.push(name); } - used_extrinsic_time.push((slope, name)); - } - }); - reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { - if !slope.is_zero() { - if !used_components.contains(&name) { used_components.push(name); } - used_reads.push((slope, name)); - } - }); - writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { - if !slope.is_zero() { - if !used_components.contains(&name) { used_components.push(name); } - used_writes.push((slope, name)); - } - }); - - let all_components = batch.results[0].components - .iter() - .map(|(name, _)| -> String { return name.to_string() }) - .collect::>(); - - // function name - write!(file, "{}fn {}(", indent, benchmark_string)?; - // params - for component in all_components { - if used_components.contains(&&component) { - write!(file, "{}: u32, ", component)?; - } else { - write!(file, "_{}: u32, ", component)?; - } - } - // return value - write!(file, ") -> Weight {{\n")?; - - write!(file, "{}{}({} as Weight)\n", indent, indent, underscore(extrinsic_time.base.saturating_mul(1000)))?; - used_extrinsic_time.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { - write!( - file, - "{}{}{}.saturating_add(({} as Weight).saturating_mul({} as Weight))\n", - indent, indent, indent, - underscore(slope.saturating_mul(1000)), - name, - ) - })?; - - if !reads.base.is_zero() { - write!( - file, - "{}{}{}.saturating_add(T::DbWeight::get().reads({} as Weight))\n", - indent, indent, indent, - reads.base, - )?; - } - used_reads.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { - write!( - file, - "{}{}{}.saturating_add(T::DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", - indent, indent, indent, - slope, - name, - ) - })?; - - if !writes.base.is_zero() { - write!( - file, - "{}{}{}.saturating_add(T::DbWeight::get().writes({} as Weight))\n", - indent, indent, indent, - writes.base, - )?; - } - used_writes.iter().try_for_each(|(slope, name)| -> Result<(), std::io::Error> { - write!( - file, - "{}{}{}.saturating_add(T::DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", - indent, indent, indent, - slope, - name, - ) - })?; - - // close function - write!(file, "{}}}\n", indent)?; - - // Check if this is the end of the iterator - if let Some(next) = batches_iter.peek() { - // Next pallet is different than current pallet, so we close up the file and open a new one. - if next.pallet != current_pallet { - write!(file, "}}\n")?; - let next_pallet = String::from_utf8(next.pallet.clone()).unwrap(); - - let mut file_path = path.clone(); - file_path.push(next_pallet); - file_path.set_extension("rs"); - file = open_file(file_path)?; - } +// A helper to join a string of vectors. +#[derive(Clone, Copy)] +struct JoinHelper; +impl handlebars::HelperDef for JoinHelper { + fn call<'reg: 'rc, 'rc>( + &self, h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).unwrap(); + let value = param.value(); + let joined = if value.is_array() { + value.as_array().unwrap() + .iter() + .map(|v| v.render()) + .collect::>() + .join(" ") } else { - // This is the end of the iterator, so we close up the final file. - write!(file, "}}\n")?; + value.render() + }; + out.write(&joined)?; + Ok(()) + } +} + +// u128 does not serialize well into JSON for `handlebars`, so we represent it as a string. +fn string_serialize(x: &u128, s: S) -> Result +where + S: serde::Serializer, +{ + s.serialize_str(&x.to_string()) +} + +#[cfg(test)] +mod test { + use super::*; + use frame_benchmarking::{BenchmarkBatch, BenchmarkParameter, BenchmarkResults}; + + fn test_data(pallet: &[u8], benchmark: &[u8], param: BenchmarkParameter, base: u32, slope: u32) -> BenchmarkBatch { + let mut results = Vec::new(); + for i in 0 .. 5 { + results.push( + BenchmarkResults { + components: vec![(param, i), (BenchmarkParameter::z, 0)], + extrinsic_time: (base + slope * i).into(), + storage_root_time: (base + slope * i).into(), + reads: (base + slope * i).into(), + repeat_reads: 0, + writes: (base + slope * i).into(), + repeat_writes: 0, + } + ) + } + + return BenchmarkBatch { + pallet: [pallet.to_vec(), b"_pallet".to_vec()].concat(), + benchmark: [benchmark.to_vec(), b"_benchmark".to_vec()].concat(), + results, } } - Ok(()) + fn check_data(benchmark: &BenchmarkData, component: &str, base: u128, slope: u128) { + assert_eq!( + benchmark.components, + vec![ + Component { name: component.to_string(), is_used: true }, + Component { name: "z".to_string(), is_used: false}, + ], + ); + // Weights multiplied by 1,000 + assert_eq!(benchmark.base_weight, base * 1_000); + assert_eq!( + benchmark.component_weight, + vec![ComponentSlope { name: component.to_string(), slope: slope * 1_000 }] + ); + // DB Reads/Writes are untouched + assert_eq!(benchmark.base_reads, base); + assert_eq!( + benchmark.component_reads, + vec![ComponentSlope { name: component.to_string(), slope: slope }] + ); + assert_eq!(benchmark.base_writes, base); + assert_eq!( + benchmark.component_writes, + vec![ComponentSlope { name: component.to_string(), slope: slope }] + ); + } + + #[test] + fn map_results_works() { + let mapped_results = map_results(&[ + test_data(b"first", b"first", BenchmarkParameter::a, 10, 3), + test_data(b"first", b"second", BenchmarkParameter::b, 9, 2), + test_data(b"second", b"first", BenchmarkParameter::c, 3, 4), + ]).unwrap(); + + let first_benchmark = &mapped_results.get("first_pallet").unwrap()[0]; + assert_eq!(first_benchmark.name, "first_benchmark"); + check_data(first_benchmark, "a", 10, 3); + + let second_benchmark = &mapped_results.get("first_pallet").unwrap()[1]; + assert_eq!(second_benchmark.name, "second_benchmark"); + check_data(second_benchmark, "b", 9, 2); + + let second_pallet_benchmark = &mapped_results.get("second_pallet").unwrap()[0]; + assert_eq!(second_pallet_benchmark.name, "first_benchmark"); + check_data(second_pallet_benchmark, "c", 3, 4); + } }