Make election benchmarks more *memory-aware* (#9286)

* Make benchmarks a bit better with mem * Make election benchmarks more *memory-aware* * Fix a few errors * cargo run --release --features=runtime-benchmarks --manifest-path=bin/node/cli/Cargo.toml -- benchmark --chain=dev --steps=50 --repeat=20 --pallet=pallet_election_provider_multi_phase --extrinsic=* --execution=wasm --wasm-execution=compiled --heap-pages=4096 --output=./frame/election-provider-multi-phase/src/weights.rs --template=./.maintain/frame-weight-template.hbs * Manually fix the weights * Update lock file * remove dupe * Fix tests * cargo update pwasm Co-authored-by: Parity Bot <admin@parity.io>
2026-06-19 14:41:02 +00:00 · 2021-07-09 21:55:31 +02:00
parent 9fc86cb55f
commit 3850a43323
11 changed files with 356 additions and 148 deletions
@@ -20,10 +20,9 @@
 use super::*;
 use crate::{Pallet as MultiPhase, unsigned::IndexAssignmentOf};
 use frame_benchmarking::{account, impl_benchmark_test_suite};
-use frame_support::{assert_ok, traits::OnInitialize};
+use frame_support::{assert_ok, traits::Hooks};
 use frame_system::RawOrigin;
 use rand::{prelude::SliceRandom, rngs::SmallRng, SeedableRng};
-use frame_election_provider_support::Assignment;
 use sp_arithmetic::{per_things::Percent, traits::One};
 use sp_npos_elections::IndexAssignment;
 use sp_runtime::InnerOf;
@@ -38,14 +37,14 @@ fn solution_with_size<T: Config>(
 	size: SolutionOrSnapshotSize,
 	active_voters_count: u32,
 	desired_targets: u32,
-) -> RawSolution<CompactOf<T>> {
-	assert!(size.targets >= desired_targets, "must have enough targets");
-	assert!(
+) -> Result<RawSolution<CompactOf<T>>, &'static str> {
+	ensure!(size.targets >= desired_targets, "must have enough targets");
+	ensure!(
 		size.targets >= (<CompactOf<T>>::LIMIT * 2) as u32,
 		"must have enough targets for unique votes."
 	);
-	assert!(size.voters >= active_voters_count, "must have enough voters");
-	assert!(
+	ensure!(size.voters >= active_voters_count, "must have enough voters");
+	ensure!(
 		(<CompactOf<T>>::LIMIT as u32) < desired_targets,
 		"must have enough winners to give them votes."
 	);
@@ -125,7 +124,7 @@ fn solution_with_size<T: Config>(
 		.map(|(voter, _stake, votes)| {
 			let percent_per_edge: InnerOf<CompactAccuracyOf<T>> =
 				(100 / votes.len()).try_into().unwrap_or_else(|_| panic!("failed to convert"));
-			Assignment {
+			crate::unsigned::Assignment::<T> {
 				who: voter.clone(),
 				distribution: votes
 					.iter()
@@ -141,7 +140,31 @@ fn solution_with_size<T: Config>(
 	let round = <MultiPhase<T>>::round();

 	assert!(score[0] > 0, "score is zero, this probably means that the stakes are not set.");
-	RawSolution { compact, score, round }
+	Ok(RawSolution { compact, score, round })
+}
+
+fn set_up_data_provider<T: Config>(v: u32, t: u32) {
+	// number of votes in snapshot.
+
+	T::DataProvider::clear();
+	log!(info, "setting up with voters = {} [degree = {}], targets = {}", v, T::DataProvider::MAXIMUM_VOTES_PER_VOTER, t);
+
+	// fill targets.
+	let mut targets = (0..t).map(|i| {
+		let target = frame_benchmarking::account::<T::AccountId>("Target", i, SEED);
+		T::DataProvider::add_target(target.clone());
+		target
+	}).collect::<Vec<_>>();
+	// we should always have enough voters to fill.
+	assert!(targets.len() > T::DataProvider::MAXIMUM_VOTES_PER_VOTER as usize);
+	targets.truncate(T::DataProvider::MAXIMUM_VOTES_PER_VOTER as usize);
+
+	// fill voters.
+	(0..v).for_each(|i| {
+		let voter = frame_benchmarking::account::<T::AccountId>("Voter", i, SEED);
+		let weight = T::Currency::minimum_balance().saturated_into::<u64>() * 1000;
+		T::DataProvider::add_voter(voter, weight, targets.clone());
+	});
 }

 frame_benchmarking::benchmarks! {
@@ -223,14 +246,18 @@ frame_benchmarking::benchmarks! {

 	// a call to `<Pallet as ElectionProvider>::elect` where we only return the queued solution.
 	elect_queued {
-		// assume largest values for the election status. These will merely affect the decoding.
-		let v = T::BenchmarkingConfig::VOTERS[1];
-		let t = T::BenchmarkingConfig::TARGETS[1];
-		let a = T::BenchmarkingConfig::ACTIVE_VOTERS[1];
-		let d = T::BenchmarkingConfig::DESIRED_TARGETS[1];
+		// number of votes in snapshot.
+		let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1];
+		// number of targets in snapshot.
+		let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1];
+		// number of assignments, i.e. compact.len(). This means the active nominators, thus must be
+		// a subset of `v` component.
+		let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1];
+		// number of desired targets. Must be a subset of `t` component.
+		let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1];

 		let witness = SolutionOrSnapshotSize { voters: v, targets: t };
-		let raw_solution = solution_with_size::<T>(witness, a, d);
+		let raw_solution = solution_with_size::<T>(witness, a, d)?;
 		let ready_solution =
 			<MultiPhase<T>>::feasibility_check(raw_solution, ElectionCompute::Signed).unwrap();

@@ -251,15 +278,6 @@ frame_benchmarking::benchmarks! {
 		assert_eq!(<CurrentPhase<T>>::get(), <Phase<T::BlockNumber>>::Off);
 	}

-	#[extra]
-	create_snapshot {
-		assert!(<MultiPhase<T>>::snapshot().is_none());
-	}: {
-		<MultiPhase::<T>>::create_snapshot().unwrap()
-	} verify {
-		assert!(<MultiPhase<T>>::snapshot().is_some());
-	}
-
 	submit {
 		let c in 1 .. (T::SignedMaxSubmissions::get() - 1);

@@ -307,7 +325,7 @@ frame_benchmarking::benchmarks! {
 			T::BenchmarkingConfig::DESIRED_TARGETS[1];

 		let witness = SolutionOrSnapshotSize { voters: v, targets: t };
-		let raw_solution = solution_with_size::<T>(witness, a, d);
+		let raw_solution = solution_with_size::<T>(witness, a, d)?;

 		assert!(<MultiPhase<T>>::queued_solution().is_none());
 		<CurrentPhase<T>>::put(Phase::Unsigned((true, 1u32.into())));
@@ -324,6 +342,84 @@ frame_benchmarking::benchmarks! {
 		assert!(<MultiPhase<T>>::queued_solution().is_some());
 	}

+	// This is checking a valid solution. The worse case is indeed a valid solution.
+	feasibility_check {
+		// number of votes in snapshot.
+		let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1];
+		// number of targets in snapshot.
+		let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1];
+		// number of assignments, i.e. compact.len(). This means the active nominators, thus must be
+		// a subset of `v` component.
+		let a in (T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1];
+		// number of desired targets. Must be a subset of `t` component.
+		let d in (T::BenchmarkingConfig::DESIRED_TARGETS[0]) .. T::BenchmarkingConfig::DESIRED_TARGETS[1];
+
+		let size = SolutionOrSnapshotSize { voters: v, targets: t };
+		let raw_solution = solution_with_size::<T>(size, a, d)?;
+
+		assert_eq!(raw_solution.compact.voter_count() as u32, a);
+		assert_eq!(raw_solution.compact.unique_targets().len() as u32, d);
+
+		// encode the most significant storage item that needs to be decoded in the dispatch.
+		let encoded_snapshot = <MultiPhase<T>>::snapshot().unwrap().encode();
+	}: {
+		assert_ok!(<MultiPhase<T>>::feasibility_check(raw_solution, ElectionCompute::Unsigned));
+		let _decoded_snap = <RoundSnapshot<T::AccountId> as Decode>::decode(&mut &*encoded_snapshot).unwrap();
+	}
+
+	// NOTE: this weight is not used anywhere, but the fact that it should succeed when execution in
+	// isolation is vital to ensure memory-safety. For the same reason, we don't care about the
+	// components iterating, we merely check that this operation will work with the "maximum"
+	// numbers.
+	//
+	// ONLY run this benchmark in isolation, and pass the `--extra` flag to enable it.
+	//
+	// NOTE: If this benchmark does not run out of memory with a given heap pages, it means that the
+	// OCW process can SURELY succeed with the given configuration, but the opposite is not true.
+	// This benchmark is doing more work than a raw call to `OffchainWorker_offchain_worker` runtime
+	// api call, since it is also setting up some mock data, which will itself exhaust the heap to
+	// some extent.
+	#[extra]
+	mine_solution_offchain_memory {
+		// number of votes in snapshot. Fixed to maximum.
+		let v = T::BenchmarkingConfig::MINER_MAXIMUM_VOTERS;
+		// number of targets in snapshot. Fixed to maximum.
+		let t  = T::BenchmarkingConfig::MAXIMUM_TARGETS;
+
+		T::DataProvider::clear();
+		set_up_data_provider::<T>(v, t);
+		let now = frame_system::Pallet::<T>::block_number();
+		<CurrentPhase<T>>::put(Phase::Unsigned((true, now)));
+		<MultiPhase::<T>>::create_snapshot().unwrap();
+	}: {
+		// we can't really verify this as it won't write anything to state, check logs.
+		<MultiPhase::<T>>::offchain_worker(now)
+	}
+
+	// NOTE: this weight is not used anywhere, but the fact that it should succeed when execution in
+	// isolation is vital to ensure memory-safety. For the same reason, we don't care about the
+	// components iterating, we merely check that this operation will work with the "maximum"
+	// numbers.
+	//
+	// ONLY run this benchmark in isolation, and pass the `--extra` flag to enable it.
+	#[extra]
+	create_snapshot_memory {
+		// number of votes in snapshot. Fixed to maximum.
+		let v = T::BenchmarkingConfig::SNAPSHOT_MAXIMUM_VOTERS;
+		// number of targets in snapshot. Fixed to maximum.
+		let t  = T::BenchmarkingConfig::MAXIMUM_TARGETS;
+
+		T::DataProvider::clear();
+		set_up_data_provider::<T>(v, t);
+		assert!(<MultiPhase<T>>::snapshot().is_none());
+	}: {
+		<MultiPhase::<T>>::create_snapshot().unwrap()
+	} verify {
+		assert!(<MultiPhase<T>>::snapshot().is_some());
+		assert_eq!(<MultiPhase<T>>::snapshot_metadata().unwrap().voters, v + t);
+		assert_eq!(<MultiPhase<T>>::snapshot_metadata().unwrap().targets, t);
+	}
+
 	#[extra]
 	trim_assignments_length {
 		// number of votes in snapshot.
@@ -344,7 +440,7 @@ frame_benchmarking::benchmarks! {
 		// Compute a random solution, then work backwards to get the lists of voters, targets, and
 		// assignments
 		let witness = SolutionOrSnapshotSize { voters: v, targets: t };
-		let RawSolution { compact, .. } = solution_with_size::<T>(witness, a, d);
+		let RawSolution { compact, .. } = solution_with_size::<T>(witness, a, d)?;
 		let RoundSnapshot { voters, targets } = MultiPhase::<T>::snapshot().unwrap();
 		let voter_at = helpers::voter_at_fn::<T>(&voters);
 		let target_at = helpers::target_at_fn::<T>(&targets);
@@ -394,39 +490,10 @@ frame_benchmarking::benchmarks! {
 		log!(trace, "actual encoded size = {}", encoding.len());
 		assert!(encoding.len() <= desired_size);
 	}
-
-	// This is checking a valid solution. The worse case is indeed a valid solution.
-	feasibility_check {
-		// number of votes in snapshot.
-		let v in (T::BenchmarkingConfig::VOTERS[0]) .. T::BenchmarkingConfig::VOTERS[1];
-		// number of targets in snapshot.
-		let t in (T::BenchmarkingConfig::TARGETS[0]) .. T::BenchmarkingConfig::TARGETS[1];
-		// number of assignments, i.e. compact.len(). This means the active nominators, thus must be
-		// a subset of `v` component.
-		let a in
-			(T::BenchmarkingConfig::ACTIVE_VOTERS[0]) .. T::BenchmarkingConfig::ACTIVE_VOTERS[1];
-		// number of desired targets. Must be a subset of `t` component.
-		let d in
-			(T::BenchmarkingConfig::DESIRED_TARGETS[0]) ..
-			T::BenchmarkingConfig::DESIRED_TARGETS[1];
-
-		let size = SolutionOrSnapshotSize { voters: v, targets: t };
-		let raw_solution = solution_with_size::<T>(size, a, d);
-
-		assert_eq!(raw_solution.compact.voter_count() as u32, a);
-		assert_eq!(raw_solution.compact.unique_targets().len() as u32, d);
-
-		// encode the most significant storage item that needs to be decoded in the dispatch.
-		let encoded_snapshot = <MultiPhase<T>>::snapshot().unwrap().encode();
-	}: {
-		assert_ok!(<MultiPhase<T>>::feasibility_check(raw_solution, ElectionCompute::Unsigned));
-		let _decoded_snap = <RoundSnapshot<T::AccountId> as Decode>::decode(&mut &*encoded_snapshot)
-			.unwrap();
-	}
 }

 impl_benchmark_test_suite!(
 	MultiPhase,
-	crate::mock::ExtBuilder::default().build(),
+	crate::mock::ExtBuilder::default().build_offchainify(10).0,
 	crate::mock::Runtime,
 );