Custom Benchmark Errors and Override (#9517)

* initial idea

* update benchmark test to frame v2

* fix some errors

* fixes for elec phrag

* fix tests

* update extrinsic time and docs

* fix import

* undo extra changes

* helper function

* wrong way

* Update frame/benchmarking/src/utils.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* doesnt need encode/decode

* fix benchmark return

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/frame/benchmarking/src/analysis.rs

@@ -17,7 +17,7 @@
 
 //! Tools for analyzing the benchmark results.
 
-use crate::BenchmarkResults;
+use crate::BenchmarkResult;
 use core::convert::TryFrom;
 use linregress::{FormulaRegressionBuilder, RegressionDataBuilder};
 use std::collections::BTreeMap;
@@ -76,7 +76,7 @@ impl TryFrom<Option<String>> for AnalysisChoice {
 impl Analysis {
 	// Useful for when there are no components, and we just need an median value of the benchmark
 	// results. Note: We choose the median value because it is more robust to outliers.
-	fn median_value(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
+	fn median_value(r: &Vec<BenchmarkResult>, selector: BenchmarkSelector) -> Option<Self> {
 		if r.is_empty() {
 			return None
 		}
@@ -104,7 +104,7 @@ impl Analysis {
 		})
 	}
 
-	pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
+	pub fn median_slopes(r: &Vec<BenchmarkResult>, selector: BenchmarkSelector) -> Option<Self> {
 		if r[0].components.is_empty() {
 			return Self::median_value(r, selector)
 		}
@@ -199,7 +199,7 @@ impl Analysis {
 		})
 	}
 
-	pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
+	pub fn min_squares_iqr(r: &Vec<BenchmarkResult>, selector: BenchmarkSelector) -> Option<Self> {
 		if r[0].components.is_empty() {
 			return Self::median_value(r, selector)
 		}
@@ -279,7 +279,7 @@ impl Analysis {
 		})
 	}
 
-	pub fn max(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
+	pub fn max(r: &Vec<BenchmarkResult>, selector: BenchmarkSelector) -> Option<Self> {
 		let median_slopes = Self::median_slopes(r, selector);
 		let min_squares = Self::min_squares_iqr(r, selector);
 
@@ -402,8 +402,8 @@ mod tests {
 		storage_root_time: u128,
 		reads: u32,
 		writes: u32,
-	) -> BenchmarkResults {
+	) -> BenchmarkResult {
-		BenchmarkResults {
+		BenchmarkResult {
 			components,
 			extrinsic_time,
 			storage_root_time,

substrate/frame/benchmarking/src/lib.rs

@@ -644,7 +644,7 @@ macro_rules! benchmark_backend {
 				&self,
 				components: &[($crate::BenchmarkParameter, u32)],
 				verify: bool
-			) -> Result<$crate::Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
+			) -> Result<$crate::Box<dyn FnOnce() -> Result<(), $crate::BenchmarkError>>, &'static str> {
 				$(
 					// Prepare instance
 					let $param = components.iter()
@@ -658,7 +658,7 @@ macro_rules! benchmark_backend {
 				$( $param_instancer ; )*
 				$( $post )*
 
-				Ok($crate::Box::new(move || -> Result<(), &'static str> {
+				Ok($crate::Box::new(move || -> Result<(), $crate::BenchmarkError> {
 					$eval;
 					if verify {
 						$postcode;
@@ -717,7 +717,7 @@ macro_rules! selected_benchmark {
 				&self,
 				components: &[($crate::BenchmarkParameter, u32)],
 				verify: bool
-			) -> Result<$crate::Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str> {
+			) -> Result<$crate::Box<dyn FnOnce() -> Result<(), $crate::BenchmarkError>>, &'static str> {
 				match self {
 					$(
 						Self::$bench => <
@@ -741,7 +741,7 @@ macro_rules! impl_benchmark {
 		( $( $name_skip_meta:ident ),* )
 	) => {
 		impl<T: Config $(<$instance>, $instance: $instance_bound )? >
-			$crate::Benchmarking<$crate::BenchmarkResults> for Pallet<T $(, $instance)? >
+			$crate::Benchmarking for Pallet<T $(, $instance)? >
 			where T: frame_system::Config, $( $where_clause )*
 		{
 			fn benchmarks(extra: bool) -> $crate::Vec<$crate::BenchmarkMetadata> {
@@ -772,13 +772,13 @@ macro_rules! impl_benchmark {
 				whitelist: &[$crate::TrackedStorageKey],
 				verify: bool,
 				internal_repeats: u32,
-			) -> Result<$crate::Vec<$crate::BenchmarkResults>, &'static str> {
+			) -> Result<$crate::Vec<$crate::BenchmarkResult>, $crate::BenchmarkError> {
 				// Map the input to the selected benchmark.
 				let extrinsic = $crate::sp_std::str::from_utf8(extrinsic)
 					.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
 				let selected_benchmark = match extrinsic {
 					$( stringify!($name) => SelectedBenchmark::$name, )*
-					_ => return Err("Could not find extrinsic."),
+					_ => return Err("Could not find extrinsic.".into()),
 				};
 
 				// Add whitelist to DB including whitelisted caller
@@ -790,7 +790,7 @@ macro_rules! impl_benchmark {
 				whitelist.push(whitelisted_caller_key.into());
 				$crate::benchmarking::set_whitelist(whitelist);
 
-				let mut results: $crate::Vec<$crate::BenchmarkResults> = $crate::Vec::new();
+				let mut results: $crate::Vec<$crate::BenchmarkResult> = $crate::Vec::new();
 
 				// Always do at least one internal repeat...
 				for _ in 0 .. internal_repeats.max(1) {
@@ -852,13 +852,13 @@ macro_rules! impl_benchmark {
 					let elapsed_storage_root = finish_storage_root - start_storage_root;
 
 					let skip_meta = [ $( stringify!($name_skip_meta).as_ref() ),* ];
-					let read_and_written_keys = if (&skip_meta).contains(&extrinsic) {
+					let read_and_written_keys = if skip_meta.contains(&extrinsic) {
 						$crate::vec![(b"Skipped Metadata".to_vec(), 0, 0, false)]
 					} else {
 						$crate::benchmarking::get_read_and_written_keys()
 					};
 
-					results.push($crate::BenchmarkResults {
+					results.push($crate::BenchmarkResult {
 						components: c.to_vec(),
 						extrinsic_time: elapsed_extrinsic,
 						storage_root_time: elapsed_storage_root,
@@ -893,14 +893,14 @@ macro_rules! impl_benchmark {
 			/// by the `impl_benchmark_test_suite` macro. However, it is not an error if a pallet
 			/// author chooses not to implement benchmarks.
 			#[allow(unused)]
-			fn test_bench_by_name(name: &[u8]) -> Result<(), &'static str> {
+			fn test_bench_by_name(name: &[u8]) -> Result<(), $crate::BenchmarkError> {
 				let name = $crate::sp_std::str::from_utf8(name)
-					.map_err(|_| "`name` is not a valid utf8 string!")?;
+					.map_err(|_| -> $crate::BenchmarkError { "`name` is not a valid utf8 string!".into() })?;
 				match name {
 					$( stringify!($name) => {
 						$crate::paste::paste! { Self::[< test_benchmark_ $name >]() }
 					} )*
-					_ => Err("Could not find test for requested benchmark."),
+					_ => Err("Could not find test for requested benchmark.".into()),
 				}
 			}
 		}
@@ -925,7 +925,7 @@ macro_rules! impl_benchmark_test {
 			where T: frame_system::Config, $( $where_clause )*
 			{
 				#[allow(unused)]
-				fn [<test_benchmark_ $name>] () -> Result<(), &'static str> {
+				fn [<test_benchmark_ $name>] () -> Result<(), $crate::BenchmarkError> {
 					let selected_benchmark = SelectedBenchmark::$name;
 					let components = <
 						SelectedBenchmark as $crate::BenchmarkingSetup<T, _>
@@ -933,7 +933,7 @@ macro_rules! impl_benchmark_test {
 
 					let execute_benchmark = |
 						c: $crate::Vec<($crate::BenchmarkParameter, u32)>
-					| -> Result<(), &'static str> {
+					| -> Result<(), $crate::BenchmarkError> {
 						// Set up the benchmark, return execution + verification function.
 						let closure_to_verify = <
 							SelectedBenchmark as $crate::BenchmarkingSetup<T, _>
@@ -1213,8 +1213,15 @@ macro_rules! impl_benchmark_test_suite {
 								anything_failed = true;
 							},
 							Ok(Err(err)) => {
-								println!("{}: {}", String::from_utf8_lossy(benchmark_name), err);
+								match err {
-								anything_failed = true;
+									$crate::BenchmarkError::Stop(err) => {
+										println!("{}: {:?}", String::from_utf8_lossy(benchmark_name), err);
+										anything_failed = true;
+									},
+									$crate::BenchmarkError::Override(_) => {
+										// This is still considered a success condition.
+									}
+								}
 							},
 							Ok(Ok(_)) => (),
 						}
@@ -1328,25 +1335,40 @@ macro_rules! add_benchmark {
 			internal_repeats,
 		} = config;
 		if &pallet[..] == &name_string[..] {
-			$batches.push($crate::BenchmarkBatch {
+			let benchmark_result = $( $location )*::run_benchmark(
-				pallet: name_string.to_vec(),
+				&benchmark[..],
-				instance: instance_string.to_vec(),
+				&selected_components[..],
-				benchmark: benchmark.clone(),
+				whitelist,
-				results: $( $location )*::run_benchmark(
+				*verify,
-					&benchmark[..],
+				*internal_repeats,
-					&selected_components[..],
+			);
-					whitelist,
+
-					*verify,
+			let final_results = match benchmark_result {
-					*internal_repeats,
+				Ok(results) => results,
-				).map_err(|e| {
+				Err($crate::BenchmarkError::Override(mut result)) => {
+					// Insert override warning as the first storage key.
+					result.keys.insert(0,
+						(b"Benchmark Override".to_vec(), 0, 0, false)
+					);
+					$crate::vec![result]
+				},
+				Err($crate::BenchmarkError::Stop(e)) => {
 					$crate::show_benchmark_debug_info(
 						instance_string,
 						benchmark,
 						selected_components,
 						verify,
 						e,
-					)
+					);
-				})?
+					return Err(e.into());
+				},
+			};
+
+			$batches.push($crate::BenchmarkBatch {
+				pallet: name_string.to_vec(),
+				instance: instance_string.to_vec(),
+				benchmark: benchmark.clone(),
+				results: final_results,
 			});
 		}
 	)

substrate/frame/benchmarking/src/tests.rs

@@ -28,48 +28,45 @@ use sp_runtime::{
 };
 use sp_std::prelude::*;
 
+#[frame_support::pallet]
 mod pallet_test {
-	use frame_support::pallet_prelude::Get;
+	use frame_support::pallet_prelude::*;
+	use frame_system::pallet_prelude::*;
 
-	frame_support::decl_storage! {
+	#[pallet::pallet]
-		trait Store for Module<T: Config> as Test where
+	#[pallet::generate_store(pub(super) trait Store)]
-			<T as OtherConfig>::OtherEvent: Into<<T as Config>::Event>
+	pub struct Pallet<T>(_);
-		{
-			pub Value get(fn value): Option<u32>;
-		}
-	}
 
-	frame_support::decl_module! {
+	#[pallet::config]
-		pub struct Module<T: Config> for enum Call where
+	pub trait Config: frame_system::Config {
-			origin: T::Origin, <T as OtherConfig>::OtherEvent: Into<<T as Config>::Event>
-		{
-			#[weight = 0]
-			fn set_value(origin, n: u32) -> frame_support::dispatch::DispatchResult {
-				let _sender = frame_system::ensure_signed(origin)?;
-				Value::put(n);
-				Ok(())
-			}
-
-			#[weight = 0]
-			fn dummy(origin, _n: u32) -> frame_support::dispatch::DispatchResult {
-				let _sender = frame_system::ensure_none(origin)?;
-				Ok(())
-			}
-		}
-	}
-
-	pub trait OtherConfig {
-		type OtherEvent;
-	}
-
-	pub trait Config: frame_system::Config + OtherConfig
-	where
-		Self::OtherEvent: Into<<Self as Config>::Event>,
-	{
-		type Event;
 		type LowerBound: Get<u32>;
 		type UpperBound: Get<u32>;
 	}
+
+	#[pallet::storage]
+	#[pallet::getter(fn heartbeat_after)]
+	pub(crate) type Value<T: Config> = StorageValue<_, u32, OptionQuery>;
+
+	#[pallet::call]
+	impl<T: Config> Pallet<T> {
+		#[pallet::weight(0)]
+		pub fn set_value(origin: OriginFor<T>, n: u32) -> DispatchResult {
+			let _sender = frame_system::ensure_signed(origin)?;
+			Value::<T>::put(n);
+			Ok(())
+		}
+
+		#[pallet::weight(0)]
+		pub fn dummy(origin: OriginFor<T>, _n: u32) -> DispatchResult {
+			let _sender = frame_system::ensure_none(origin)?;
+			Ok(())
+		}
+
+		#[pallet::weight(0)]
+		pub fn always_error(_origin: OriginFor<T>) -> DispatchResult {
+			return Err("I always fail".into())
+		}
+	}
 }
 
 type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic<Test>;
@@ -118,27 +115,18 @@ parameter_types! {
 }
 
 impl pallet_test::Config for Test {
-	type Event = Event;
 	type LowerBound = LowerBound;
 	type UpperBound = UpperBound;
 }
 
-impl pallet_test::OtherConfig for Test {
-	type OtherEvent = Event;
-}
-
 fn new_test_ext() -> sp_io::TestExternalities {
 	GenesisConfig::default().build_storage().unwrap().into()
 }
 
 mod benchmarks {
-	use super::{
+	use super::{new_test_ext, pallet_test::Value, Test};
-		new_test_ext,
+	use crate::{account, BenchmarkError, BenchmarkParameter, BenchmarkResult, BenchmarkingSetup};
-		pallet_test::{self, Value},
+	use frame_support::{assert_err, assert_ok, ensure, traits::Get};
-		Test,
-	};
-	use crate::{account, BenchmarkParameter, BenchmarkingSetup};
-	use frame_support::{assert_err, assert_ok, ensure, traits::Get, StorageValue};
 	use frame_system::RawOrigin;
 	use sp_std::prelude::*;
 
@@ -148,8 +136,7 @@ mod benchmarks {
 	crate::benchmarks! {
 		where_clause {
 			where
-				<T as pallet_test::OtherConfig>::OtherEvent: Into<<T as pallet_test::Config>::Event> + Clone,
+				crate::tests::Origin: From<RawOrigin<<T as frame_system::Config>::AccountId>>,
-				<T as pallet_test::Config>::Event: Clone,
 		}
 
 		set_value {
@@ -157,7 +144,7 @@ mod benchmarks {
 			let caller = account::<T::AccountId>("caller", 0, 0);
 		}: _ (RawOrigin::Signed(caller), b.into())
 		verify {
-			assert_eq!(Value::get(), Some(b));
+			assert_eq!(Value::<T>::get(), Some(b));
 		}
 
 		other_name {
@@ -206,7 +193,7 @@ mod benchmarks {
 			let caller = account::<T::AccountId>("caller", 0, 0);
 		}: set_value(RawOrigin::Signed(caller), b.into())
 		verify {
-			assert_eq!(Value::get(), Some(b));
+			assert_eq!(Value::<T>::get(), Some(b));
 		}
 
 		#[skip_meta]
@@ -215,7 +202,21 @@ mod benchmarks {
 			let caller = account::<T::AccountId>("caller", 0, 0);
 		}: set_value(RawOrigin::Signed(caller), b.into())
 		verify {
-			assert_eq!(Value::get(), Some(b));
+			assert_eq!(Value::<T>::get(), Some(b));
+		}
+
+		override_benchmark {
+			let b in 1 .. 1000;
+			let caller = account::<T::AccountId>("caller", 0, 0);
+		}: {
+			Err(BenchmarkError::Override(
+				BenchmarkResult {
+					extrinsic_time: 1_234_567_890,
+					reads: 1337,
+					writes: 420,
+					..Default::default()
+				}
+			))?;
 		}
 	}
 
@@ -306,6 +307,23 @@ mod benchmarks {
 		});
 	}
 
+	#[test]
+	fn benchmark_override_works() {
+		let selected = SelectedBenchmark::override_benchmark;
+
+		let closure = <SelectedBenchmark as BenchmarkingSetup<Test>>::instance(
+			&selected,
+			&[(BenchmarkParameter::b, 1)],
+			true,
+		)
+		.expect("failed to create closure");
+
+		new_test_ext().execute_with(|| {
+			let result = closure();
+			assert!(matches!(result, Err(BenchmarkError::Override(_))));
+		});
+	}
+
 	#[test]
 	fn benchmarks_generate_unit_tests() {
 		new_test_ext().execute_with(|| {

substrate/frame/benchmarking/src/utils.rs

@@ -18,7 +18,11 @@
 //! Interfaces, types and utils for benchmarking a FRAME runtime.
 
 use codec::{Decode, Encode};
-use frame_support::traits::StorageInfo;
+use frame_support::{
+	dispatch::{DispatchError, DispatchErrorWithPostInfo},
+	pallet_prelude::*,
+	traits::StorageInfo,
+};
 use sp_io::hashing::blake2_256;
 use sp_std::{prelude::Box, vec::Vec};
 use sp_storage::TrackedStorageKey;
@@ -73,7 +77,7 @@ pub struct BenchmarkBatch {
 	/// The extrinsic (or benchmark name) of this benchmark.
 	pub benchmark: Vec<u8>,
 	/// The results from this benchmark.
-	pub results: Vec<BenchmarkResults>,
+	pub results: Vec<BenchmarkResult>,
 }
 
 // TODO: could probably make API cleaner here.
@@ -87,16 +91,16 @@ pub struct BenchmarkBatchSplitResults {
 	/// The extrinsic (or benchmark name) of this benchmark.
 	pub benchmark: Vec<u8>,
 	/// The extrinsic timing results from this benchmark.
-	pub time_results: Vec<BenchmarkResults>,
+	pub time_results: Vec<BenchmarkResult>,
 	/// The db tracking results from this benchmark.
-	pub db_results: Vec<BenchmarkResults>,
+	pub db_results: Vec<BenchmarkResult>,
 }
 
-/// Results from running benchmarks on a FRAME pallet.
+/// Result from running benchmarks on a FRAME pallet.
 /// Contains duration of the function call in nanoseconds along with the benchmark parameters
 /// used for that benchmark result.
 #[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
-pub struct BenchmarkResults {
+pub struct BenchmarkResult {
 	pub components: Vec<(BenchmarkParameter, u32)>,
 	pub extrinsic_time: u128,
 	pub storage_root_time: u128,
@@ -108,6 +112,50 @@ pub struct BenchmarkResults {
 	pub keys: Vec<(Vec<u8>, u32, u32, bool)>,
 }
 
+impl BenchmarkResult {
+	pub fn from_weight(w: Weight) -> Self {
+		Self { extrinsic_time: (w as u128) / 1_000, ..Default::default() }
+	}
+}
+
+/// Possible errors returned from the benchmarking pipeline.
+///
+/// * Stop: The benchmarking pipeline should stop and return the inner string.
+/// * WeightOverride: The benchmarking pipeline is allowed to fail here, and we should use the
+///   included weight instead.
+#[derive(Clone, PartialEq, Debug)]
+pub enum BenchmarkError {
+	Stop(&'static str),
+	Override(BenchmarkResult),
+}
+
+impl From<BenchmarkError> for &'static str {
+	fn from(e: BenchmarkError) -> Self {
+		match e {
+			BenchmarkError::Stop(s) => s,
+			BenchmarkError::Override(_) => "benchmark override",
+		}
+	}
+}
+
+impl From<&'static str> for BenchmarkError {
+	fn from(s: &'static str) -> Self {
+		Self::Stop(s)
+	}
+}
+
+impl From<DispatchErrorWithPostInfo> for BenchmarkError {
+	fn from(e: DispatchErrorWithPostInfo) -> Self {
+		Self::Stop(e.into())
+	}
+}
+
+impl From<DispatchError> for BenchmarkError {
+	fn from(e: DispatchError) -> Self {
+		Self::Stop(e.into())
+	}
+}
+
 /// Configuration used to setup and run runtime benchmarks.
 #[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
 pub struct BenchmarkConfig {
@@ -235,7 +283,7 @@ pub trait Benchmarking {
 }
 
 /// The pallet benchmarking trait.
-pub trait Benchmarking<T> {
+pub trait Benchmarking {
 	/// Get the benchmarks available for this pallet. Generally there is one benchmark per
 	/// extrinsic, so these are sometimes just called "extrinsics".
 	///
@@ -251,7 +299,7 @@ pub trait Benchmarking<T> {
 		whitelist: &[TrackedStorageKey],
 		verify: bool,
 		internal_repeats: u32,
-	) -> Result<Vec<T>, &'static str>;
+	) -> Result<Vec<BenchmarkResult>, BenchmarkError>;
 }
 
 /// The required setup for creating a benchmark.
@@ -267,7 +315,7 @@ pub trait BenchmarkingSetup<T, I = ()> {
 		&self,
 		components: &[(BenchmarkParameter, u32)],
 		verify: bool,
-	) -> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>;
+	) -> Result<Box<dyn FnOnce() -> Result<(), BenchmarkError>>, &'static str>;
 }
 
 /// Grab an account, seeded by a name and index.

substrate/frame/collective/src/benchmarking.rs

@@ -277,7 +277,7 @@ benchmarks_instance! {
 	verify {
 		// All proposals exist and the last proposal has just been updated.
 		assert_eq!(Collective::<T, _>::proposals().len(), p as usize);
-		let voting = Collective::<T, _>::voting(&last_hash).ok_or(Error::<T, I>::ProposalMissing)?;
+		let voting = Collective::<T, _>::voting(&last_hash).ok_or("Proposal Missing")?;
 		assert_eq!(voting.ayes.len(), (m - 3) as usize);
 		assert_eq!(voting.nays.len(), 1);
 	}

substrate/frame/democracy/src/benchmarking.rs

@@ -136,7 +136,7 @@ benchmarks! {
 		}
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Votes were not recorded.");
 
@@ -146,7 +146,7 @@ benchmarks! {
 	verify {
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r + 1) as usize, "Vote was not recorded.");
 	}
@@ -164,7 +164,7 @@ benchmarks! {
 		}
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded.");
 
@@ -179,14 +179,14 @@ benchmarks! {
 	verify {
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r + 1) as usize, "Vote was incorrectly added");
 		let referendum_info = Democracy::<T>::referendum_info(referendum_index)
 			.ok_or("referendum doesn't exist")?;
 		let tally =  match referendum_info {
 			ReferendumInfo::Ongoing(r) => r.tally,
-			_ => return Err("referendum not ongoing"),
+			_ => return Err("referendum not ongoing".into()),
 		};
 		assert_eq!(tally.nays, 1000u32.into(), "changed vote was not recorded");
 	}
@@ -374,7 +374,7 @@ benchmarks! {
 			if let Some(value) = ReferendumInfoOf::<T>::get(i) {
 				match value {
 					ReferendumInfo::Finished { .. } => (),
-					ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished"),
+					ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished".into()),
 				}
 			}
 		}
@@ -408,7 +408,7 @@ benchmarks! {
 			if let Some(value) = ReferendumInfoOf::<T>::get(i) {
 				match value {
 					ReferendumInfo::Finished { .. } => (),
-					ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished"),
+					ReferendumInfo::Ongoing(_) => return Err("Referendum was not finished".into()),
 				}
 			}
 		}
@@ -438,7 +438,7 @@ benchmarks! {
 		for i in 0 .. r {
 			if let Some(value) = ReferendumInfoOf::<T>::get(i) {
 				match value {
-					ReferendumInfo::Finished { .. } => return Err("Referendum has been finished"),
+					ReferendumInfo::Finished { .. } => return Err("Referendum has been finished".into()),
 					ReferendumInfo::Ongoing(_) => (),
 				}
 			}
@@ -462,7 +462,7 @@ benchmarks! {
 		)?;
 		let (target, balance) = match VotingOf::<T>::get(&caller) {
 			Voting::Delegating { target, balance, .. } => (target, balance),
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(target, old_delegate, "delegation target didn't work");
 		assert_eq!(balance, delegated_balance, "delegation balance didn't work");
@@ -476,7 +476,7 @@ benchmarks! {
 		}
 		let votes = match VotingOf::<T>::get(&new_delegate) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Votes were not recorded.");
 		whitelist_account!(caller);
@@ -484,13 +484,13 @@ benchmarks! {
 	verify {
 		let (target, balance) = match VotingOf::<T>::get(&caller) {
 			Voting::Delegating { target, balance, .. } => (target, balance),
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(target, new_delegate, "delegation target didn't work");
 		assert_eq!(balance, delegated_balance, "delegation balance didn't work");
 		let delegations = match VotingOf::<T>::get(&new_delegate) {
 			Voting::Direct { delegations, .. } => delegations,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(delegations.capital, delegated_balance, "delegation was not recorded.");
 	}
@@ -512,7 +512,7 @@ benchmarks! {
 		)?;
 		let (target, balance) = match VotingOf::<T>::get(&caller) {
 			Voting::Delegating { target, balance, .. } => (target, balance),
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(target, the_delegate, "delegation target didn't work");
 		assert_eq!(balance, delegated_balance, "delegation balance didn't work");
@@ -528,7 +528,7 @@ benchmarks! {
 		}
 		let votes = match VotingOf::<T>::get(&the_delegate) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Votes were not recorded.");
 		whitelist_account!(caller);
@@ -537,7 +537,7 @@ benchmarks! {
 		// Voting should now be direct
 		match VotingOf::<T>::get(&caller) {
 			Voting::Direct { .. } => (),
-			_ => return Err("undelegation failed"),
+			_ => return Err("undelegation failed".into()),
 		}
 	}
 
@@ -558,7 +558,7 @@ benchmarks! {
 		let proposal_hash = T::Hashing::hash(&encoded_proposal[..]);
 		match Preimages::<T>::get(proposal_hash) {
 			Some(PreimageStatus::Available { .. }) => (),
-			_ => return Err("preimage not available")
+			_ => return Err("preimage not available".into())
 		}
 	}
 
@@ -580,7 +580,7 @@ benchmarks! {
 		let proposal_hash = T::Hashing::hash(&encoded_proposal[..]);
 		match Preimages::<T>::get(proposal_hash) {
 			Some(PreimageStatus::Available { .. }) => (),
-			_ => return Err("preimage not available")
+			_ => return Err("preimage not available".into())
 		}
 	}
 
@@ -652,7 +652,7 @@ benchmarks! {
 
 		let votes = match VotingOf::<T>::get(&locker) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r + 1) as usize, "Votes were not recorded.");
 
@@ -667,7 +667,7 @@ benchmarks! {
 	verify {
 		let votes = match VotingOf::<T>::get(&locker) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Vote was not removed");
 
@@ -689,7 +689,7 @@ benchmarks! {
 
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Votes not created");
 
@@ -699,7 +699,7 @@ benchmarks! {
 	verify {
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed");
 	}
@@ -718,7 +718,7 @@ benchmarks! {
 
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), r as usize, "Votes not created");
 
@@ -728,7 +728,7 @@ benchmarks! {
 	verify {
 		let votes = match VotingOf::<T>::get(&caller) {
 			Voting::Direct { votes, .. } => votes,
-			_ => return Err("Votes are not direct"),
+			_ => return Err("Votes are not direct".into()),
 		};
 		assert_eq!(votes.len(), (r - 1) as usize, "Vote was not removed");
 	}
@@ -747,7 +747,7 @@ benchmarks! {
 
 		match Preimages::<T>::get(proposal_hash) {
 			Some(PreimageStatus::Available { .. }) => (),
-			_ => return Err("preimage not available")
+			_ => return Err("preimage not available".into())
 		}
 	}: enact_proposal(RawOrigin::Root, proposal_hash, 0)
 	verify {
@@ -768,7 +768,7 @@ benchmarks! {
 
 		match Preimages::<T>::get(proposal_hash) {
 			Some(PreimageStatus::Available { .. }) => (),
-			_ => return Err("preimage not available")
+			_ => return Err("preimage not available".into())
 		}
 	}: {
 		assert_eq!(

substrate/frame/election-provider-multi-phase/src/benchmarking.rs

@@ -438,7 +438,7 @@ frame_benchmarking::benchmarks! {
 		set_up_data_provider::<T>(v, t);
 		assert!(<MultiPhase<T>>::snapshot().is_none());
 	}: {
-		<MultiPhase::<T>>::create_snapshot()?
+		<MultiPhase::<T>>::create_snapshot().map_err(|_| "could not create snapshot")?;
 	} verify {
 		assert!(<MultiPhase<T>>::snapshot().is_some());
 		assert_eq!(<MultiPhase<T>>::snapshot_metadata().ok_or("snapshot missing")?.voters, v + t);

substrate/frame/elections-phragmen/src/benchmarking.rs

@@ -21,7 +21,9 @@
 
 use super::*;
 
-use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelist};
+use frame_benchmarking::{
+	account, benchmarks, impl_benchmark_test_suite, whitelist, BenchmarkError, BenchmarkResult,
+};
 use frame_support::{dispatch::DispatchResultWithPostInfo, traits::OnInitialize};
 use frame_system::RawOrigin;
 
@@ -332,9 +334,16 @@ benchmarks! {
 		}
 	}
 
+	// We use the max block weight for this extrinsic for now. See below.
+	remove_member_without_replacement {}: {
+		Err(BenchmarkError::Override(
+			BenchmarkResult::from_weight(T::BlockWeights::get().max_block)
+		))?;
+	}
+
 	// -- Root ones
 	#[extra] // this calls into phragmen and consumes a full block for now.
-	remove_member_without_replacement {
+	remove_member_without_replacement_extra {
 		// worse case is when we remove a member and we have no runner as a replacement. This
 		// triggers phragmen again. The only parameter is how many candidates will compete for the
 		// new slot.

substrate/frame/elections-phragmen/src/lib.rs

@@ -474,7 +474,7 @@ pub mod pallet {
 		#[pallet::weight(if *has_replacement {
 			T::WeightInfo::remove_member_with_replacement()
 		} else {
-			T::BlockWeights::get().max_block
+			T::WeightInfo::remove_member_without_replacement()
 		})]
 		pub fn remove_member(
 			origin: OriginFor<T>,

substrate/frame/elections-phragmen/src/weights.rs

@@ -54,6 +54,7 @@ pub trait WeightInfo {
 	fn renounce_candidacy_members() -> Weight;
 	fn renounce_candidacy_runners_up() -> Weight;
 	fn remove_member_with_replacement() -> Weight;
+	fn remove_member_without_replacement() -> Weight;
 	fn remove_member_wrong_refund() -> Weight;
 	fn clean_defunct_voters(v: u32, d: u32, ) -> Weight;
 	fn election_phragmen(c: u32, v: u32, e: u32, ) -> Weight;
@@ -150,6 +151,9 @@ impl<T: frame_system::Config> WeightInfo for SubstrateWeight<T> {
 			.saturating_add(T::DbWeight::get().reads(5 as Weight))
 			.saturating_add(T::DbWeight::get().writes(5 as Weight))
 	}
+	fn remove_member_without_replacement() -> Weight {
+		T::BlockWeights::get().max_block
+	}
 	// Storage: Elections RunnersUp (r:1 w:0)
 	fn remove_member_wrong_refund() -> Weight {
 		(6_697_000 as Weight)
@@ -282,6 +286,11 @@ impl WeightInfo for () {
 			.saturating_add(RocksDbWeight::get().reads(5 as Weight))
 			.saturating_add(RocksDbWeight::get().writes(5 as Weight))
 	}
+	fn remove_member_without_replacement() -> Weight {
+		(76_153_000 as Weight)
+			.saturating_add(RocksDbWeight::get().reads(5 as Weight))
+			.saturating_add(RocksDbWeight::get().writes(5 as Weight))
+	}
 	// Storage: Elections RunnersUp (r:1 w:0)
 	fn remove_member_wrong_refund() -> Weight {
 		(6_697_000 as Weight)

substrate/frame/im-online/src/benchmarking.rs

@@ -82,7 +82,8 @@ benchmarks! {
 		let (input_heartbeat, signature) = create_heartbeat::<T>(k, e)?;
 		let call = Call::heartbeat(input_heartbeat, signature);
 	}: {
-		ImOnline::<T>::validate_unsigned(TransactionSource::InBlock, &call)?;
+		ImOnline::<T>::validate_unsigned(TransactionSource::InBlock, &call)
+			.map_err(|e| -> &'static str { e.into() })?;
 	}
 
 	validate_unsigned_and_then_heartbeat {
@@ -91,7 +92,8 @@ benchmarks! {
 		let (input_heartbeat, signature) = create_heartbeat::<T>(k, e)?;
 		let call = Call::heartbeat(input_heartbeat, signature);
 	}: {
-		ImOnline::<T>::validate_unsigned(TransactionSource::InBlock, &call)?;
+		ImOnline::<T>::validate_unsigned(TransactionSource::InBlock, &call)
+			.map_err(|e| -> &'static str { e.into() })?;
 		call.dispatch_bypass_filter(RawOrigin::None.into())?;
 	}
 }

substrate/utils/frame/benchmarking-cli/src/command.rs

@@ -19,7 +19,7 @@ use crate::BenchmarkCmd;
 use codec::{Decode, Encode};
 use frame_benchmarking::{
 	Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter,
-	BenchmarkResults, BenchmarkSelector,
+	BenchmarkResult, BenchmarkSelector,
 };
 use frame_support::traits::StorageInfo;
 use linked_hash_map::LinkedHashMap;
@@ -48,7 +48,7 @@ fn combine_batches(
 	}
 
 	let mut all_benchmarks =
-		LinkedHashMap::<_, (Vec<BenchmarkResults>, Vec<BenchmarkResults>)>::new();
+		LinkedHashMap::<_, (Vec<BenchmarkResult>, Vec<BenchmarkResult>)>::new();
 
 	db_batches
 		.into_iter()

substrate/utils/frame/benchmarking-cli/src/writer.rs

@@ -29,7 +29,7 @@ use serde::Serialize;
 
 use crate::BenchmarkCmd;
 use frame_benchmarking::{
-	Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResults, BenchmarkSelector,
+	Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector,
 	RegressionModel,
 };
 use frame_support::traits::StorageInfo;
@@ -359,7 +359,7 @@ pub fn write_results(
 // each benchmark.
 fn add_storage_comments(
 	comments: &mut Vec<String>,
-	results: &[BenchmarkResults],
+	results: &[BenchmarkResult],
 	storage_info: &[StorageInfo],
 ) {
 	let mut storage_info_map = storage_info
@@ -377,6 +377,16 @@ fn add_storage_comments(
 	};
 	storage_info_map.insert(skip_storage_info.prefix.clone(), &skip_storage_info);
 
+	// Special hack to show `Benchmark Override`
+	let benchmark_override = StorageInfo {
+		pallet_name: b"Benchmark".to_vec(),
+		storage_name: b"Override".to_vec(),
+		prefix: b"Benchmark Override".to_vec(),
+		max_values: None,
+		max_size: None,
+	};
+	storage_info_map.insert(benchmark_override.prefix.clone(), &benchmark_override);
+
 	// This tracks the keys we already identified, so we only generate a single comment.
 	let mut identified = HashSet::<Vec<u8>>::new();
 
@@ -502,7 +512,7 @@ where
 #[cfg(test)]
 mod test {
 	use super::*;
-	use frame_benchmarking::{BenchmarkBatchSplitResults, BenchmarkParameter, BenchmarkResults};
+	use frame_benchmarking::{BenchmarkBatchSplitResults, BenchmarkParameter, BenchmarkResult};
 
 	fn test_data(
 		pallet: &[u8],
@@ -513,7 +523,7 @@ mod test {
 	) -> BenchmarkBatchSplitResults {
 		let mut results = Vec::new();
 		for i in 0..5 {
-			results.push(BenchmarkResults {
+			results.push(BenchmarkResult {
 				components: vec![(param, i), (BenchmarkParameter::z, 0)],
 				extrinsic_time: (base + slope * i).into(),
 				storage_root_time: (base + slope * i).into(),