Add runtime benchmarking suite to all runtimes (#1034)

* Add benchmarking suite to all runtimes

* Add `runtime-benchmarks` feature to `test-linux-stable`

* Update Cargo.lock

* Update Cargo.lock

* Update Cargo.lock

* "user" instead of "caller"

* undo these changes

polkadot/.gitlab-ci.yml

@@ -126,7 +126,7 @@ test-linux-stable:                 &test
     RUSTFLAGS: -Cdebug-assertions=y
     TARGET: native
   script:
-    - time cargo test --all --release --verbose --locked
+    - time cargo test --all --release --verbose --locked --features runtime-benchmarks
     - sccache -s
 
 
@@ -281,4 +281,3 @@ deploy-polkasync-kusama:
     POLKADOT_CI_COMMIT_REF:        "${CI_COMMIT_REF}"
   allow_failure:                   true
   trigger:                         "parity/infrastructure/parity-testnet"
-

polkadot/runtime/common/src/claims.rs

@@ -445,7 +445,7 @@ mod tests {
 
 	// This function basically just builds a genesis storage key/value store according to
 	// our desired mockup.
-	fn new_test_ext() -> sp_io::TestExternalities {
+	pub fn new_test_ext() -> sp_io::TestExternalities {
 		let mut t = system::GenesisConfig::default().build_storage::<Test>().unwrap();
 		// We use default for brevity, but you can configure as desired if needed.
 		balances::GenesisConfig::<Test>::default().assimilate_storage(&mut t).unwrap();
@@ -678,14 +678,21 @@ mod benchmarking {
 			let vesting = Some((100_000.into(), 1_000.into(), 100.into()));
 			let signature = sig::<T>(&secret_key, &account.encode());
 			super::Module::<T>::mint_claim(RawOrigin::Root.into(), eth_address, VALUE.into(), vesting)?;
+			assert_eq!(Claims::<T>::get(eth_address), Some(VALUE.into()));
 		}: _(RawOrigin::None, account, signature)
+		verify {
+			assert_eq!(Claims::<T>::get(eth_address), None);
+		}
 
 		// Benchmark `mint_claim` when there already exists `c` claims in storage.
 		mint_claim {
 			let c in ...;
-			let account = account("user", c, SEED);
+			let eth_address = account("eth_address", c, SEED);
 			let vesting = Some((100_000.into(), 1_000.into(), 100.into()));
-		}: _(RawOrigin::Root, account, VALUE.into(), vesting)
+		}: _(RawOrigin::Root, eth_address, VALUE.into(), vesting)
+		verify {
+			assert_eq!(Claims::<T>::get(eth_address), Some(VALUE.into()));
+		}
 
 		// Benchmark the time it takes to execute `validate_unsigned`
 		validate_unsigned {
@@ -720,8 +727,26 @@ mod benchmarking {
 			let data = account.using_encoded(to_ascii_hex);
 		}: {
 			for _ in 0 .. i {
-				let _maybe_signer = super::Module::<T>::eth_recover(&signature, &data);
+				assert!(super::Module::<T>::eth_recover(&signature, &data).is_some());
 			}
 		}
 	}
+
+	#[cfg(test)]
+	mod tests {
+		use super::*;
+		use crate::claims::tests::{new_test_ext, Test};
+		use frame_support::assert_ok;
+
+		#[test]
+		fn test_benchmarks() {
+			new_test_ext().execute_with(|| {
+				assert_ok!(test_benchmark_claim::<Test>());
+				assert_ok!(test_benchmark_mint_claim::<Test>());
+				assert_ok!(test_benchmark_validate_unsigned::<Test>());
+				assert_ok!(test_benchmark_keccak256::<Test>());
+				assert_ok!(test_benchmark_eth_recover::<Test>());
+			});
+		}
+	}
 }

polkadot/runtime/kusama/Cargo.toml

@@ -60,7 +60,10 @@ timestamp = { package = "pallet-timestamp", git = "https://github.com/paritytech
 treasury = { package = "pallet-treasury", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 utility = { package = "pallet-utility", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 vesting = { package = "pallet-vesting", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-offences-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 
 runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false }
 primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
@@ -137,11 +140,23 @@ std = [
 	"runtime-common/std",
 ]
 runtime-benchmarks = [
-	"collective/runtime-benchmarks",
+	"runtime-common/runtime-benchmarks",
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
-	"runtime-common/runtime-benchmarks",
-	"elections-phragmen/runtime-benchmarks",
-	"society/runtime-benchmarks",
 	"system/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+	"balances/runtime-benchmarks",
+	"collective/runtime-benchmarks",
+	"democracy/runtime-benchmarks",
+	"elections-phragmen/runtime-benchmarks",
+	"identity/runtime-benchmarks",
+	"im-online/runtime-benchmarks",
+	"society/runtime-benchmarks",
+	"staking/runtime-benchmarks",
+	"timestamp/runtime-benchmarks",
+	"treasury/runtime-benchmarks",
+	"utility/runtime-benchmarks",
+	"vesting/runtime-benchmarks",
+	"pallet-offences-benchmarking",
+	"pallet-session-benchmarking",
 ]

polkadot/runtime/kusama/src/lib.rs

@@ -1010,10 +1010,33 @@ sp_api::impl_runtime_apis! {
 			repeat: u32,
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, RuntimeString> {
 			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
+			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
+			// To get around that, we separated the Session benchmarks into its own crate, which is why
+			// we need these two lines below.
+			use pallet_session_benchmarking::Module as SessionBench;
+			use pallet_offences_benchmarking::Module as OffencesBench;
+
+			impl pallet_session_benchmarking::Trait for Runtime {}
+			impl pallet_offences_benchmarking::Trait for Runtime {}
 
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
+			// Polkadot
 			add_benchmark!(params, batches, b"claims", Claims);
+			// Substrate
+			add_benchmark!(params, batches, b"balances", Balances);
+			add_benchmark!(params, batches, b"collective", Council);
+			add_benchmark!(params, batches, b"democracy", Democracy);
+			add_benchmark!(params, batches, b"identity", Identity);
+			add_benchmark!(params, batches, b"im-online", ImOnline);
+			add_benchmark!(params, batches, b"offences", OffencesBench::<Runtime>);
+			add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
+			add_benchmark!(params, batches, b"staking", Staking);
+			add_benchmark!(params, batches, b"timestamp", Timestamp);
+			add_benchmark!(params, batches, b"treasury", Treasury);
+			add_benchmark!(params, batches, b"utility", Utility);
+			add_benchmark!(params, batches, b"vesting", Vesting);
+
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)
 		}

polkadot/runtime/polkadot/Cargo.toml

@@ -57,7 +57,10 @@ timestamp = { package = "pallet-timestamp", git = "https://github.com/paritytech
 treasury = { package = "pallet-treasury", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sudo = { package = "pallet-sudo", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 vesting = { package = "pallet-vesting", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-offences-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 
 runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false }
 primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
@@ -131,10 +134,20 @@ std = [
 	"vesting/std",
 ]
 runtime-benchmarks = [
-	"collective/runtime-benchmarks",
+	"runtime-common/runtime-benchmarks",
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
-	"runtime-common/runtime-benchmarks",
-	"elections-phragmen/runtime-benchmarks",
 	"system/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+	"balances/runtime-benchmarks",
+	"collective/runtime-benchmarks",
+	"democracy/runtime-benchmarks",
+	"elections-phragmen/runtime-benchmarks",
+	"im-online/runtime-benchmarks",
+	"staking/runtime-benchmarks",
+	"timestamp/runtime-benchmarks",
+	"treasury/runtime-benchmarks",
+	"vesting/runtime-benchmarks",
+	"pallet-offences-benchmarking",
+	"pallet-session-benchmarking",
 ]

polkadot/runtime/polkadot/src/lib.rs

@@ -925,10 +925,31 @@ sp_api::impl_runtime_apis! {
 			repeat: u32,
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, RuntimeString> {
 			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
+			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
+			// To get around that, we separated the Session benchmarks into its own crate, which is why
+			// we need these two lines below.
+			use pallet_session_benchmarking::Module as SessionBench;
+			use pallet_offences_benchmarking::Module as OffencesBench;
+
+			impl pallet_session_benchmarking::Trait for Runtime {}
+			impl pallet_offences_benchmarking::Trait for Runtime {}
 
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
+			// Polkadot
 			add_benchmark!(params, batches, b"claims", Claims);
+			// Substrate
+			add_benchmark!(params, batches, b"balances", Balances);
+			add_benchmark!(params, batches, b"collective", Council);
+			add_benchmark!(params, batches, b"democracy", Democracy);
+			add_benchmark!(params, batches, b"im-online", ImOnline);
+			add_benchmark!(params, batches, b"offences", OffencesBench::<Runtime>);
+			add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
+			add_benchmark!(params, batches, b"staking", Staking);
+			add_benchmark!(params, batches, b"timestamp", Timestamp);
+			add_benchmark!(params, batches, b"treasury", Treasury);
+			add_benchmark!(params, batches, b"vesting", Vesting);
+
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)
 		}

polkadot/runtime/westend/Cargo.toml

@@ -61,7 +61,10 @@ timestamp = { package = "pallet-timestamp", git = "https://github.com/paritytech
 treasury = { package = "pallet-treasury", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 utility = { package = "pallet-utility", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 vesting = { package = "pallet-vesting", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-offences-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
+pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 
 runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false }
 primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
@@ -141,11 +144,23 @@ std = [
 	"runtime-common/std",
 ]
 runtime-benchmarks = [
-	"collective/runtime-benchmarks",
+	"runtime-common/runtime-benchmarks",
 	"frame-benchmarking",
 	"frame-support/runtime-benchmarks",
-	"runtime-common/runtime-benchmarks",
-	"elections-phragmen/runtime-benchmarks",
-	"society/runtime-benchmarks",
 	"system/runtime-benchmarks",
+	"sp-runtime/runtime-benchmarks",
+	"balances/runtime-benchmarks",
+	"collective/runtime-benchmarks",
+	"democracy/runtime-benchmarks",
+	"elections-phragmen/runtime-benchmarks",
+	"identity/runtime-benchmarks",
+	"im-online/runtime-benchmarks",
+	"society/runtime-benchmarks",
+	"staking/runtime-benchmarks",
+	"timestamp/runtime-benchmarks",
+	"treasury/runtime-benchmarks",
+	"utility/runtime-benchmarks",
+	"vesting/runtime-benchmarks",
+	"pallet-offences-benchmarking",
+	"pallet-session-benchmarking",
 ]

polkadot/runtime/westend/src/lib.rs

@@ -818,9 +818,28 @@ sp_api::impl_runtime_apis! {
 			repeat: u32,
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, RuntimeString> {
 			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
+			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
+			// To get around that, we separated the Session benchmarks into its own crate, which is why
+			// we need these two lines below.
+			use pallet_session_benchmarking::Module as SessionBench;
+			use pallet_offences_benchmarking::Module as OffencesBench;
+
+			impl pallet_session_benchmarking::Trait for Runtime {}
+			impl pallet_offences_benchmarking::Trait for Runtime {}
 
 			let mut batches = Vec::<BenchmarkBatch>::new();
 			let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
+
+			add_benchmark!(params, batches, b"balances", Balances);
+			add_benchmark!(params, batches, b"identity", Identity);
+			add_benchmark!(params, batches, b"im-online", ImOnline);
+			add_benchmark!(params, batches, b"offences", OffencesBench::<Runtime>);
+			add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
+			add_benchmark!(params, batches, b"staking", Staking);
+			add_benchmark!(params, batches, b"timestamp", Timestamp);
+			add_benchmark!(params, batches, b"utility", Utility);
+			add_benchmark!(params, batches, b"vesting", Vesting);
+
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)
 		}