Add Baseline FRAME Benchmarks (#9691)

* create and add baseline

* fix import

* try a different name

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* increase repeats

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* Update baseline.rs

* Update baseline.rs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* improve hash benchmark

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

* cargo run --quiet --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=frame_benchmarking --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/benchmarking/src/weights.rs --template=./.maintain/frame-weight-template.hbs

Co-authored-by: Parity Bot <admin@parity.io>

substrate/bin/node-template/runtime/src/lib.rs

@@ -463,12 +463,14 @@ impl_runtime_apis! {
 			Vec<frame_benchmarking::BenchmarkList>,
 			Vec<frame_support::traits::StorageInfo>,
 		) {
-			use frame_benchmarking::{list_benchmark, Benchmarking, BenchmarkList};
+			use frame_benchmarking::{list_benchmark, baseline, Benchmarking, BenchmarkList};
 			use frame_support::traits::StorageInfoTrait;
 			use frame_system_benchmarking::Pallet as SystemBench;
+			use baseline::Pallet as BaselineBench;
 
 			let mut list = Vec::<BenchmarkList>::new();
 
+			list_benchmark!(list, extra, frame_benchmarking, BaselineBench::<Runtime>);
 			list_benchmark!(list, extra, frame_system, SystemBench::<Runtime>);
 			list_benchmark!(list, extra, pallet_balances, Balances);
 			list_benchmark!(list, extra, pallet_timestamp, Timestamp);
@@ -482,10 +484,13 @@ impl_runtime_apis! {
 		fn dispatch_benchmark(
 			config: frame_benchmarking::BenchmarkConfig
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
-			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
+			use frame_benchmarking::{baseline, Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
 
 			use frame_system_benchmarking::Pallet as SystemBench;
+			use baseline::Pallet as BaselineBench;
+
 			impl frame_system_benchmarking::Config for Runtime {}
+			impl baseline::Config for Runtime {}
 
 			let whitelist: Vec<TrackedStorageKey> = vec![
 				// Block Number
@@ -503,6 +508,7 @@ impl_runtime_apis! {
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&config, &whitelist);
 
+			add_benchmark!(params, batches, frame_benchmarking, BaselineBench::<Runtime>);
 			add_benchmark!(params, batches, frame_system, SystemBench::<Runtime>);
 			add_benchmark!(params, batches, pallet_balances, Balances);
 			add_benchmark!(params, batches, pallet_timestamp, Timestamp);

substrate/bin/node/runtime/src/lib.rs

@@ -1609,7 +1609,7 @@ impl_runtime_apis! {
 			Vec<frame_benchmarking::BenchmarkList>,
 			Vec<frame_support::traits::StorageInfo>,
 		) {
-			use frame_benchmarking::{list_benchmark, Benchmarking, BenchmarkList};
+			use frame_benchmarking::{list_benchmark, baseline, Benchmarking, BenchmarkList};
 			use frame_support::traits::StorageInfoTrait;
 
 			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency
@@ -1618,9 +1618,11 @@ impl_runtime_apis! {
 			use pallet_session_benchmarking::Pallet as SessionBench;
 			use pallet_offences_benchmarking::Pallet as OffencesBench;
 			use frame_system_benchmarking::Pallet as SystemBench;
+			use baseline::Pallet as BaselineBench;
 
 			let mut list = Vec::<BenchmarkList>::new();
 
+			list_benchmark!(list, extra, frame_benchmarking, BaselineBench::<Runtime>);
 			list_benchmark!(list, extra, pallet_assets, Assets);
 			list_benchmark!(list, extra, pallet_babe, Babe);
 			list_benchmark!(list, extra, pallet_bags_list, BagsList);
@@ -1662,7 +1664,7 @@ impl_runtime_apis! {
 		fn dispatch_benchmark(
 			config: frame_benchmarking::BenchmarkConfig
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
-			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
+			use frame_benchmarking::{baseline, Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
 
 			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency
 			// issues. To get around that, we separated the Session benchmarks into its own crate,
@@ -1670,10 +1672,12 @@ impl_runtime_apis! {
 			use pallet_session_benchmarking::Pallet as SessionBench;
 			use pallet_offences_benchmarking::Pallet as OffencesBench;
 			use frame_system_benchmarking::Pallet as SystemBench;
+			use baseline::Pallet as BaselineBench;
 
 			impl pallet_session_benchmarking::Config for Runtime {}
 			impl pallet_offences_benchmarking::Config for Runtime {}
 			impl frame_system_benchmarking::Config for Runtime {}
+			impl baseline::Config for Runtime {}
 
 			let whitelist: Vec<TrackedStorageKey> = vec![
 				// Block Number
@@ -1695,6 +1699,7 @@ impl_runtime_apis! {
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&config, &whitelist);
 
+			add_benchmark!(params, batches, frame_benchmarking, BaselineBench::<Runtime>);
 			add_benchmark!(params, batches, pallet_assets, Assets);
 			add_benchmark!(params, batches, pallet_babe, Babe);
 			add_benchmark!(params, batches, pallet_balances, Balances);

substrate/frame/benchmarking/src/baseline.rs

@@ -0,0 +1,173 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! A set of benchmarks which can establish a global baseline for all other
+//! benchmarking.
+
+use crate::benchmarks;
+use codec::Encode;
+use frame_system::Pallet as System;
+use sp_runtime::traits::Hash;
+use sp_std::prelude::*;
+
+pub struct Pallet<T: Config>(System<T>);
+pub trait Config: frame_system::Config {}
+
+benchmarks! {
+	addition {
+		let i in 0 .. 1_000_000;
+		let mut start = 0;
+	}: {
+		(0..i).for_each(|_| start += 1);
+	} verify {
+		assert_eq!(start, i);
+	}
+
+	subtraction {
+		let i in 0 .. 1_000_000;
+		let mut start = u32::MAX;
+	}: {
+		(0..i).for_each(|_| start -= 1);
+	} verify {
+		assert_eq!(start, u32::MAX - i);
+	}
+
+	multiplication {
+		let i in 0 .. 1_000_000;
+		let mut out = 0;
+	}:  {
+		(1..=i).for_each(|j| out = 2 * j);
+	} verify {
+		assert_eq!(out, 2 * i);
+	}
+
+	division {
+		let i in 0 .. 1_000_000;
+		let mut out = 0;
+	}: {
+		(0..=i).for_each(|j| out = j / 2);
+	} verify {
+		assert_eq!(out, i / 2);
+	}
+
+	hashing {
+		let i in 0 .. 100;
+		let mut hash = T::Hash::default();
+	}: {
+		(0..=100_000u32).for_each(|j| hash = T::Hashing::hash(&j.to_be_bytes()));
+	} verify {
+		assert!(hash != T::Hash::default());
+	}
+
+	#[skip_meta]
+	storage_read {
+		let i in 0 .. 1_000;
+		let mut people = Vec::new();
+		(0..i).for_each(|j| {
+			let hash = T::Hashing::hash(&j.to_be_bytes()).encode();
+			frame_support::storage::unhashed::put(&hash, &hash);
+			people.push(hash);
+		});
+	}: {
+		people.iter().for_each(|hash| {
+			// This does a storage read
+			let value = frame_support::storage::unhashed::get(hash);
+			assert_eq!(value, Some(hash.to_vec()));
+		});
+	}
+
+	#[skip_meta]
+	storage_write {
+		let i in 0 .. 1_000;
+		let mut hashes = Vec::new();
+		(0..i).for_each(|j| {
+			let hash = T::Hashing::hash(&j.to_be_bytes());
+			hashes.push(hash.encode());
+		});
+	}: {
+		hashes.iter().for_each(|hash| {
+			// This does a storage write
+			frame_support::storage::unhashed::put(hash, hash);
+		});
+	} verify {
+		hashes.iter().for_each(|hash| {
+			let value = frame_support::storage::unhashed::get(hash);
+			assert_eq!(value, Some(hash.to_vec()));
+		});
+	}
+
+	impl_benchmark_test_suite!(
+		Pallet,
+		crate::baseline::mock::new_test_ext(),
+		crate::baseline::mock::Test,
+	);
+}
+
+#[cfg(test)]
+pub mod mock {
+	use sp_runtime::{testing::H256, traits::IdentityLookup};
+
+	type AccountId = u64;
+	type AccountIndex = u32;
+	type BlockNumber = u64;
+
+	type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic<Test>;
+	type Block = frame_system::mocking::MockBlock<Test>;
+
+	frame_support::construct_runtime!(
+		pub enum Test where
+			Block = Block,
+			NodeBlock = Block,
+			UncheckedExtrinsic = UncheckedExtrinsic,
+		{
+			System: frame_system::{Pallet, Call, Config, Storage, Event<T>},
+		}
+	);
+
+	impl frame_system::Config for Test {
+		type BaseCallFilter = frame_support::traits::Everything;
+		type BlockWeights = ();
+		type BlockLength = ();
+		type DbWeight = ();
+		type Origin = Origin;
+		type Index = AccountIndex;
+		type BlockNumber = BlockNumber;
+		type Call = Call;
+		type Hash = H256;
+		type Hashing = ::sp_runtime::traits::BlakeTwo256;
+		type AccountId = AccountId;
+		type Lookup = IdentityLookup<Self::AccountId>;
+		type Header = sp_runtime::testing::Header;
+		type Event = Event;
+		type BlockHashCount = ();
+		type Version = ();
+		type PalletInfo = PalletInfo;
+		type AccountData = ();
+		type OnNewAccount = ();
+		type OnKilledAccount = ();
+		type SystemWeightInfo = ();
+		type SS58Prefix = ();
+		type OnSetCode = ();
+	}
+
+	impl super::Config for Test {}
+
+	pub fn new_test_ext() -> sp_io::TestExternalities {
+		let t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
+		sp_io::TestExternalities::new(t)
+	}
+}

substrate/frame/benchmarking/src/lib.rs

@@ -27,6 +27,8 @@ mod tests;
 mod tests_instance;
 mod utils;
 
+pub mod baseline;
+
 #[cfg(feature = "std")]
 pub use analysis::{Analysis, AnalysisChoice, BenchmarkSelector, RegressionModel};
 #[doc(hidden)]

substrate/frame/benchmarking/src/weights.rs

@@ -0,0 +1,122 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Autogenerated weights for frame_benchmarking
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
+//! DATE: 2021-10-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128
+
+// Executed Command:
+// target/release/substrate
+// benchmark
+// --chain=dev
+// --steps=50
+// --repeat=20
+// --pallet=frame_benchmarking
+// --extrinsic=*
+// --execution=wasm
+// --wasm-execution=compiled
+// --heap-pages=4096
+// --output=./frame/benchmarking/src/weights.rs
+// --template=./.maintain/frame-weight-template.hbs
+
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+
+use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}};
+use sp_std::marker::PhantomData;
+
+/// Weight functions needed for frame_benchmarking.
+pub trait WeightInfo {
+	fn addition(i: u32, ) -> Weight;
+	fn subtraction(i: u32, ) -> Weight;
+	fn multiplication(i: u32, ) -> Weight;
+	fn division(i: u32, ) -> Weight;
+	fn hashing(i: u32, ) -> Weight;
+	fn storage_read(i: u32, ) -> Weight;
+	fn storage_write(i: u32, ) -> Weight;
+}
+
+/// Weights for frame_benchmarking using the Substrate node and recommended hardware.
+pub struct SubstrateWeight<T>(PhantomData<T>);
+impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
+	fn addition(_i: u32, ) -> Weight {
+		(337_000 as Weight)
+	}
+	fn subtraction(_i: u32, ) -> Weight {
+		(343_000 as Weight)
+	}
+	fn multiplication(_i: u32, ) -> Weight {
+		(340_000 as Weight)
+	}
+	fn division(_i: u32, ) -> Weight {
+		(346_000 as Weight)
+	}
+	fn hashing(_i: u32, ) -> Weight {
+		(35_449_143_000 as Weight)
+	}
+	// Storage: Skipped Metadata (r:0 w:0)
+	fn storage_read(i: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 3_000
+			.saturating_add((2_851_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(i as Weight)))
+	}
+	// Storage: Skipped Metadata (r:0 w:0)
+	fn storage_write(i: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 0
+			.saturating_add((662_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
+	}
+}
+
+// For backwards compatibility and tests
+impl WeightInfo for () {
+	fn addition(_i: u32, ) -> Weight {
+		(337_000 as Weight)
+	}
+	fn subtraction(_i: u32, ) -> Weight {
+		(343_000 as Weight)
+	}
+	fn multiplication(_i: u32, ) -> Weight {
+		(340_000 as Weight)
+	}
+	fn division(_i: u32, ) -> Weight {
+		(346_000 as Weight)
+	}
+	fn hashing(_i: u32, ) -> Weight {
+		(35_449_143_000 as Weight)
+	}
+	// Storage: Skipped Metadata (r:0 w:0)
+	fn storage_read(i: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 3_000
+			.saturating_add((2_851_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(i as Weight)))
+	}
+	// Storage: Skipped Metadata (r:0 w:0)
+	fn storage_write(i: u32, ) -> Weight {
+		(0 as Weight)
+			// Standard Error: 0
+			.saturating_add((662_000 as Weight).saturating_mul(i as Weight))
+			.saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight)))
+	}
+}