FRAME Weights with Storage Metadata (#9471)

* weights with metadata

* fix

* fix contract test

* skip metadata tag

* special handling for `frame_system`

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_system --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/system/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* add skip metadata to contracts

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_contracts --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/contracts/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* fix contract test

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_democracy --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/democracy/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bounties --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bounties/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* expose component information

* fix test generation

* refactor list benchmarks

* move component selection out of runtime

* add benchmark verification

* missing feature

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bounties --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bounties/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_democracy --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/democracy/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* add internal repeats

* update weights with internal repeats

* fix warning

* return error with pov

* try without tracking

* Revert "return error with pov"

This reverts commit 44c36cbbd3c6818f36f377e3e291f1df156e40f7.

* Revert "try without tracking"

This reverts commit f401c44aebff2232389d8d307b20924891e5d77d.

* Revert "Revert "try without tracking""

This reverts commit 4b4e05929802ad3e8154e107359447634e5fb21b.

* state without tracking

* fix build

* temp test

* split db and timing benchmarks

* extend db results?

* default repeat is internal

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_democracy --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/democracy/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* fix warning

* bump linked hash map

* use linked hash map for storage tracker

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_democracy --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/democracy/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* remove conflicting short command

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_democracy --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/democracy/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* missed one linked hashmap

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bounties --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bounties/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_bounties --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/bounties/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* new weights with latest changes

* Update frame/benchmarking/src/utils.rs

Co-authored-by: Parity Benchmarking Bot <admin@parity.io>

substrate/frame/benchmarking/src/utils.rs

@@ -76,6 +76,22 @@ pub struct BenchmarkBatch {
 	pub results: Vec<BenchmarkResults>,
 }
 
+// TODO: could probably make API cleaner here.
+/// The results of a single of benchmark, where time and db results are separated.
+#[derive(Encode, Decode, Clone, PartialEq, Debug)]
+pub struct BenchmarkBatchSplitResults {
+	/// The pallet containing this benchmark.
+	pub pallet: Vec<u8>,
+	/// The instance of this pallet being benchmarked.
+	pub instance: Vec<u8>,
+	/// The extrinsic (or benchmark name) of this benchmark.
+	pub benchmark: Vec<u8>,
+	/// The extrinsic timing results from this benchmark.
+	pub time_results: Vec<BenchmarkResults>,
+	/// The db tracking results from this benchmark.
+	pub db_results: Vec<BenchmarkResults>,
+}
+
 /// Results from running benchmarks on a FRAME pallet.
 /// Contains duration of the function call in nanoseconds along with the benchmark parameters
 /// used for that benchmark result.
@@ -99,21 +115,12 @@ pub struct BenchmarkConfig {
 	pub pallet: Vec<u8>,
 	/// The encoded name of the benchmark/extrinsic to run.
 	pub benchmark: Vec<u8>,
-	/// An optional manual override to the lowest values used in the `steps` range.
-	pub lowest_range_values: Vec<u32>,
-	/// An optional manual override to the highest values used in the `steps` range.
-	pub highest_range_values: Vec<u32>,
-	/// The number of samples to take across the range of values for components. (current_step,
-	/// total_steps)
-	pub steps: (u32, u32),
-	/// The number times to repeat each benchmark to increase accuracy of results. (current_repeat,
-	/// total_repeat)
-	pub repeat: (u32, u32),
+	/// The selected component values to use when running the benchmark.
+	pub selected_components: Vec<(BenchmarkParameter, u32)>,
 	/// Enable an extra benchmark iteration which runs the verification logic for a benchmark.
 	pub verify: bool,
-	/// Enable benchmarking of "extra" extrinsics, i.e. those that are not directly used in a
-	/// pallet.
-	pub extra: bool,
+	/// Number of times to repeat benchmark within the Wasm environment. (versus in the client)
+	pub internal_repeats: u32,
 }
 
 /// A list of benchmarks available for a particular pallet and instance.
@@ -123,7 +130,13 @@ pub struct BenchmarkConfig {
 pub struct BenchmarkList {
 	pub pallet: Vec<u8>,
 	pub instance: Vec<u8>,
-	pub benchmarks: Vec<Vec<u8>>,
+	pub benchmarks: Vec<BenchmarkMetadata>,
+}
+
+#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
+pub struct BenchmarkMetadata {
+	pub name: Vec<u8>,
+	pub components: Vec<(BenchmarkParameter, u32, u32)>,
 }
 
 sp_api::decl_runtime_apis! {
@@ -228,27 +241,15 @@ pub trait Benchmarking<T> {
 	/// Parameters
 	/// - `extra`: Also return benchmarks marked "extra" which would otherwise not be
 	///            needed for weight calculation.
-	fn benchmarks(extra: bool) -> Vec<&'static [u8]>;
+	fn benchmarks(extra: bool) -> Vec<BenchmarkMetadata>;
 
 	/// Run the benchmarks for this pallet.
-	///
-	/// Parameters
-	/// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as
-	///   bytes.
-	/// - `lowest_range_values`: The lowest number for each range of parameters.
-	/// - `highest_range_values`: The highest number for each range of parameters.
-	/// - `steps`: The number of sample points you want to take across the range of parameters.
-	///   (current_step, total_steps)
-	/// - `repeat`: The total number times to repeat each benchmark to increase accuracy of results.
-	///   (current_repeat, total_repeats)
 	fn run_benchmark(
 		name: &[u8],
-		lowest_range_values: &[u32],
-		highest_range_values: &[u32],
-		steps: (u32, u32),
-		repeat: (u32, u32),
+		selected_components: &[(BenchmarkParameter, u32)],
 		whitelist: &[TrackedStorageKey],
 		verify: bool,
+		internal_repeats: u32,
 	) -> Result<Vec<T>, &'static str>;
 }