[Feature] Part 1: add TargetList for validator ranking (#12034)

* [Feature] Part 1: add TargetList for validator ranking

* remove redundant todo

* remove typo

* cleanup

* implement score

* more fixes

* fix thresholds

* fmt

* Remove the stuff that has to come in the next PR, some fixes

* extended balance import

* Change all the references from VoteWeight to Self::Score

* Add a migration for VoterBagsList

* fix score

* add targetList to nomination-pools tests

* fix bench

* address review comments

* change get_npos_targets

* address more comments

* remove thresholds for the time being

* fix instance reference

* VoterBagsListInstance

* reus

* remove params that are not used yet

* Introduced pre/post upgrade try-runtime checks

* fix

* fixes

* fix migration

* fix migration

* fix post_upgrade

* change

* Fix

* eloquent PhantomData

* fix PD

* more fixes

* Update frame/staking/src/pallet/impls.rs

Co-authored-by: Squirrel <gilescope@gmail.com>

* is_nominator now works

* fix test-staking

* build fixes

* fix remote-tests

* Apply suggestions from code review

Co-authored-by: parity-processbot <>
Co-authored-by: kianenigma <kian@parity.io>
Co-authored-by: Squirrel <gilescope@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

substrate/bin/node/runtime/src/lib.rs

@@ -568,7 +568,9 @@ impl pallet_staking::Config for Runtime {
 	type OffendingValidatorsThreshold = OffendingValidatorsThreshold;
 	type ElectionProvider = ElectionProviderMultiPhase;
 	type GenesisElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	// This a placeholder, to be introduced in the next PR as an instance of bags-list
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = NominationPools;
 	type WeightInfo = pallet_staking::weights::SubstrateWeight<Runtime>;
@@ -724,12 +726,15 @@ parameter_types! {
 	pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS;
 }
 
-impl pallet_bags_list::Config for Runtime {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
+	/// The voter bags-list is loosely kept up to date, and the real source of truth for the score
+	/// of each node is the staking pallet.
 	type ScoreProvider = Staking;
-	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
+	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 }
 
 parameter_types! {
@@ -1637,7 +1642,7 @@ construct_runtime!(
 		Gilt: pallet_gilt,
 		Uniques: pallet_uniques,
 		TransactionStorage: pallet_transaction_storage,
-		BagsList: pallet_bags_list,
+		VoterBagsList: pallet_bags_list::<Instance1>,
 		StateTrieMigration: pallet_state_trie_migration,
 		ChildBounties: pallet_child_bounties,
 		Referenda: pallet_referenda,
@@ -1723,7 +1728,7 @@ mod benches {
 		[pallet_alliance, Alliance]
 		[pallet_assets, Assets]
 		[pallet_babe, Babe]
-		[pallet_bags_list, BagsList]
+		[pallet_bags_list, VoterBagsList]
 		[pallet_balances, Balances]
 		[pallet_bounties, Bounties]
 		[pallet_child_bounties, ChildBounties]

substrate/bin/node/runtime/src/voter_bags.rs

@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +15,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Autogenerated voter bag thresholds.
+//! Autogenerated bag thresholds.
 //!
-//! Generated on 2021-07-05T09:17:40.469754927+00:00
+//! Generated on 2022-08-15T19:26:59.939787+00:00
+//! Arguments
+//! Total issuance: 100000000000000
+//! Minimum balance: 100000000000000
 //! for the node runtime.
 
 /// Existential weight for this runtime.

substrate/frame/babe/src/mock.rs

@@ -202,6 +202,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

substrate/frame/bags-list/remote-tests/src/lib.rs

@@ -18,6 +18,7 @@
 //! Utilities for remote-testing pallet-bags-list.
 
 use frame_election_provider_support::ScoreProvider;
+use pallet_bags_list::Instance1;
 use sp_std::prelude::*;
 
 /// A common log target to use.
@@ -30,18 +31,26 @@ pub mod try_state;
 /// A wrapper for a runtime that the functions of this crate expect.
 ///
 /// For example, this can be the `Runtime` type of the Polkadot runtime.
-pub trait RuntimeT:
-	pallet_staking::Config + pallet_bags_list::Config + frame_system::Config
+pub trait RuntimeT<I: 'static>:
+	pallet_staking::Config + pallet_bags_list::Config<I> + frame_system::Config
+{
+}
+impl<
+		I: 'static,
+		T: pallet_staking::Config + pallet_bags_list::Config<I> + frame_system::Config,
+	> RuntimeT<I> for T
 {
 }
-impl<T: pallet_staking::Config + pallet_bags_list::Config + frame_system::Config> RuntimeT for T {}
 
 fn percent(portion: u32, total: u32) -> f64 {
 	(portion as f64 / total as f64) * 100f64
 }
 
 /// Display the number of nodes in each bag, while identifying those that need a rebag.
-pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_name: &'static str) {
+pub fn display_and_check_bags<Runtime: RuntimeT<Instance1>>(
+	currency_unit: u64,
+	currency_name: &'static str,
+) {
 	use frame_election_provider_support::SortedListProvider;
 	use frame_support::traits::Get;
 
@@ -55,7 +64,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 	let mut seen_in_bags = 0;
 	let mut rebaggable = 0;
 	let mut active_bags = 0;
-	for vote_weight_thresh in <Runtime as pallet_bags_list::Config>::BagThresholds::get() {
+	for vote_weight_thresh in <Runtime as pallet_bags_list::Config<Instance1>>::BagThresholds::get()
+	{
 		let vote_weight_thresh_u64: u64 = (*vote_weight_thresh)
 			.try_into()
 			.map_err(|_| "runtime must configure score to at most u64 to use this test")
@@ -64,7 +74,9 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		let vote_weight_thresh_as_unit = vote_weight_thresh_u64 as f64 / currency_unit as f64;
 		let pretty_thresh = format!("Threshold: {}. {}", vote_weight_thresh_as_unit, currency_name);
 
-		let bag = match pallet_bags_list::Pallet::<Runtime>::list_bags_get(*vote_weight_thresh) {
+		let bag = match pallet_bags_list::Pallet::<Runtime, Instance1>::list_bags_get(
+			*vote_weight_thresh,
+		) {
 			Some(bag) => bag,
 			None => {
 				log::info!(target: LOG_TARGET, "{} NO VOTERS.", pretty_thresh);
@@ -75,7 +87,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		active_bags += 1;
 
 		for id in bag.std_iter().map(|node| node.std_id().clone()) {
-			let vote_weight = <Runtime as pallet_bags_list::Config>::ScoreProvider::score(&id);
+			let vote_weight =
+				<Runtime as pallet_bags_list::Config<Instance1>>::ScoreProvider::score(&id);
 			let vote_weight_thresh_u64: u64 = (*vote_weight_thresh)
 				.try_into()
 				.map_err(|_| "runtime must configure score to at most u64 to use this test")
@@ -92,8 +105,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 				);
 			}
 
-			let node =
-				pallet_bags_list::Node::<Runtime>::get(&id).expect("node in bag must exist.");
+			let node = pallet_bags_list::Node::<Runtime, Instance1>::get(&id)
+				.expect("node in bag must exist.");
 			if node.is_misplaced(vote_weight) {
 				rebaggable += 1;
 				let notional_bag = pallet_bags_list::notional_bag_for::<Runtime, _>(vote_weight);
@@ -141,7 +154,7 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		"a total of {} nodes are in {} active bags [{} total bags], {} of which can be rebagged.",
 		voter_list_count,
 		active_bags,
-		<Runtime as pallet_bags_list::Config>::BagThresholds::get().len(),
+		<Runtime as pallet_bags_list::Config<Instance1>>::BagThresholds::get().len(),
 		rebaggable,
 	);
 }

substrate/frame/bags-list/remote-tests/src/migration.rs

@@ -24,7 +24,10 @@ use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Test voter bags migration. `currency_unit` is the number of planks per the the runtimes `UNITS`
 /// (i.e. number of decimal places per DOT, KSM etc)
-pub async fn execute<Runtime: RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,

substrate/frame/bags-list/remote-tests/src/snapshot.rs

@@ -22,7 +22,10 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute create a snapshot from pallet-staking.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: crate::RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	voter_limit: Option<usize>,
 	currency_unit: u64,
 	ws_url: String,
@@ -34,7 +37,8 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 			transport: ws_url.to_string().into(),
 			// NOTE: we don't scrape pallet-staking, this kinda ensures that the source of the data
 			// is bags-list.
-			pallets: vec![pallet_bags_list::Pallet::<Runtime>::name().to_string()],
+			pallets: vec![pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::name()
+				.to_string()],
 			at: None,
 			..Default::default()
 		}))

substrate/frame/bags-list/remote-tests/src/try_state.rs

@@ -25,7 +25,10 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute the sanity check of the bags-list.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: crate::RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,
@@ -33,7 +36,8 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 	let mut ext = Builder::<Block>::new()
 		.mode(Mode::Online(OnlineConfig {
 			transport: ws_url.to_string().into(),
-			pallets: vec![pallet_bags_list::Pallet::<Runtime>::name().to_string()],
+			pallets: vec![pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::name()
+				.to_string()],
 			..Default::default()
 		}))
 		.inject_hashed_prefix(&<pallet_staking::Bonded<Runtime>>::prefix_hash())
@@ -44,7 +48,7 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 
 	ext.execute_with(|| {
 		sp_core::crypto::set_default_ss58_version(Runtime::SS58Prefix::get().try_into().unwrap());
-		pallet_bags_list::Pallet::<Runtime>::try_state().unwrap();
+		pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::try_state().unwrap();
 		log::info!(target: crate::LOG_TARGET, "executed bags-list sanity check with no errors.");
 
 		crate::display_and_check_bags::<Runtime>(currency_unit, currency_name);

substrate/frame/grandpa/src/mock.rs

@@ -206,6 +206,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

substrate/frame/nomination-pools/benchmarking/src/lib.rs

@@ -42,8 +42,11 @@ type CurrencyOf<T> = <T as pallet_nomination_pools::Config>::Currency;
 const USER_SEED: u32 = 0;
 const MAX_SPANS: u32 = 100;
 
+type VoterBagsListInstance = pallet_bags_list::Instance1;
 pub trait Config:
-	pallet_nomination_pools::Config + pallet_staking::Config + pallet_bags_list::Config
+	pallet_nomination_pools::Config
+	+ pallet_staking::Config
+	+ pallet_bags_list::Config<VoterBagsListInstance>
 {
 }
 

substrate/frame/nomination-pools/benchmarking/src/mock.rs

@@ -15,6 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::VoterBagsListInstance;
 use frame_election_provider_support::VoteWeight;
 use frame_support::{pallet_prelude::*, parameter_types, traits::ConstU64, PalletId};
 use sp_runtime::{
@@ -111,7 +112,8 @@ impl pallet_staking::Config for Runtime {
 	type ElectionProvider =
 		frame_election_provider_support::NoElection<(AccountId, BlockNumber, Staking)>;
 	type GenesisElectionProvider = Self::ElectionProvider;
-	type VoterList = pallet_bags_list::Pallet<Self>;
+	type VoterList = VoterList;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = Pools;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
@@ -122,7 +124,7 @@ parameter_types! {
 	pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
 }
 
-impl pallet_bags_list::Config for Runtime {
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
 	type BagThresholds = BagThresholds;
@@ -180,7 +182,7 @@ frame_support::construct_runtime!(
 		Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 		Pools: pallet_nomination_pools::{Pallet, Call, Storage, Event<T>},
 	}
 );

substrate/frame/nomination-pools/test-staking/src/mock.rs

@@ -126,7 +126,8 @@ impl pallet_staking::Config for Runtime {
 	type ElectionProvider =
 		frame_election_provider_support::NoElection<(AccountId, BlockNumber, Staking)>;
 	type GenesisElectionProvider = Self::ElectionProvider;
-	type VoterList = pallet_bags_list::Pallet<Self>;
+	type VoterList = VoterList;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = Pools;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
@@ -137,7 +138,8 @@ parameter_types! {
 	pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
 }
 
-impl pallet_bags_list::Config for Runtime {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
 	type BagThresholds = BagThresholds;
@@ -193,7 +195,7 @@ frame_support::construct_runtime!(
 		Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 		Pools: pallet_nomination_pools::{Pallet, Call, Storage, Event<T>},
 	}
 );

substrate/frame/offences/benchmarking/src/mock.rs

@@ -180,6 +180,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

substrate/frame/session/benchmarking/src/mock.rs

@@ -175,6 +175,7 @@ impl pallet_staking::Config for Test {
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
 	type WeightInfo = ();

substrate/frame/staking/src/benchmarking.rs

@@ -844,7 +844,7 @@ benchmarks! {
 			v, n, T::MaxNominations::get() as usize, false, None
 		)?;
 	}: {
-		let targets = <Staking<T>>::get_npos_targets();
+		let targets = <Staking<T>>::get_npos_targets(None);
 		assert_eq!(targets.len() as u32, v);
 	}
 

substrate/frame/staking/src/lib.rs

@@ -886,11 +886,12 @@ enum Releases {
 	V8_0_0,  // populate `VoterList`.
 	V9_0_0,  // inject validators into `VoterList` as well.
 	V10_0_0, // remove `EarliestUnappliedSlash`.
+	V11_0_0, // Move pallet storage prefix, e.g. BagsList -> VoterBagsList
 }
 
 impl Default for Releases {
 	fn default() -> Self {
-		Releases::V10_0_0
+		Releases::V11_0_0
 	}
 }
 

substrate/frame/staking/src/migrations.rs

@@ -20,6 +20,99 @@ use super::*;
 use frame_election_provider_support::SortedListProvider;
 use frame_support::traits::OnRuntimeUpgrade;
 
+pub mod v11 {
+	use super::*;
+	use frame_support::{
+		storage::migration::move_pallet,
+		traits::{GetStorageVersion, PalletInfoAccess},
+	};
+	#[cfg(feature = "try-runtime")]
+	use sp_io::hashing::twox_128;
+
+	pub struct MigrateToV11<T, P, N>(sp_std::marker::PhantomData<(T, P, N)>);
+	impl<T: Config, P: GetStorageVersion + PalletInfoAccess, N: Get<&'static str>> OnRuntimeUpgrade
+		for MigrateToV11<T, P, N>
+	{
+		#[cfg(feature = "try-runtime")]
+		fn pre_upgrade() -> Result<(), &'static str> {
+			frame_support::ensure!(
+				StorageVersion::<T>::get() == crate::Releases::V10_0_0,
+				"must upgrade linearly"
+			);
+			let old_pallet_prefix = twox_128(N::get().as_bytes());
+
+			frame_support::ensure!(
+				sp_io::storage::next_key(&old_pallet_prefix).is_some(),
+				"no data for the old pallet name has been detected"
+			);
+
+			Ok(())
+		}
+
+		/// Migrate the entire storage of this pallet to a new prefix.
+		///
+		/// This new prefix must be the same as the one set in construct_runtime. For safety, use
+		/// `PalletInfo` to get it, as:
+		/// `<Runtime as frame_system::Config>::PalletInfo::name::<VoterBagsList>`.
+		///
+		/// The migration will look into the storage version in order to avoid triggering a
+		/// migration on an up to date storage.
+		fn on_runtime_upgrade() -> Weight {
+			let old_pallet_name = N::get();
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+
+			if StorageVersion::<T>::get() == Releases::V10_0_0 {
+				// bump version anyway, even if we don't need to move the prefix
+				StorageVersion::<T>::put(Releases::V11_0_0);
+				if new_pallet_name == old_pallet_name {
+					log!(
+						warn,
+						"new bags-list name is equal to the old one, only bumping the version"
+					);
+					return T::DbWeight::get().reads(1).saturating_add(T::DbWeight::get().writes(1))
+				}
+
+				move_pallet(old_pallet_name.as_bytes(), new_pallet_name.as_bytes());
+				<T as frame_system::Config>::BlockWeights::get().max_block
+			} else {
+				log!(warn, "v11::migrate should be removed.");
+				T::DbWeight::get().reads(1)
+			}
+		}
+
+		#[cfg(feature = "try-runtime")]
+		fn post_upgrade() -> Result<(), &'static str> {
+			frame_support::ensure!(
+				StorageVersion::<T>::get() == crate::Releases::V11_0_0,
+				"wrong version after the upgrade"
+			);
+
+			let old_pallet_name = N::get();
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+
+			// skip storage prefix checks for the same pallet names
+			if new_pallet_name == old_pallet_name {
+				return Ok(())
+			}
+
+			let old_pallet_prefix = twox_128(N::get().as_bytes());
+			frame_support::ensure!(
+				sp_io::storage::next_key(&old_pallet_prefix).is_none(),
+				"old pallet data hasn't been removed"
+			);
+
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+			let new_pallet_prefix = twox_128(new_pallet_name.as_bytes());
+			frame_support::ensure!(
+				sp_io::storage::next_key(&new_pallet_prefix).is_some(),
+				"new pallet data hasn't been created"
+			);
+
+			Ok(())
+		}
+	}
+}
+
 pub mod v10 {
 	use super::*;
 	use frame_support::storage_alias;

substrate/frame/staking/src/mock.rs

@@ -99,7 +99,7 @@ frame_support::construct_runtime!(
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
 		Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>},
 		Historical: pallet_session::historical::{Pallet, Storage},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterBagsList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 	}
 );
 
@@ -240,9 +240,11 @@ parameter_types! {
 	pub static LedgerSlashPerEra: (BalanceOf<Test>, BTreeMap<EraIndex, BalanceOf<Test>>) = (Zero::zero(), BTreeMap::new());
 }
 
-impl pallet_bags_list::Config for Test {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Test {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
+	// Staking is the source of truth for voter bags list, since they are not kept up to date.
 	type ScoreProvider = Staking;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
@@ -296,7 +298,8 @@ impl crate::pallet::pallet::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	// NOTE: consider a macro and use `UseNominatorsAndValidatorsMap<Self>` as well.
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	type TargetList = UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = OnStakerSlashMock<Test>;
 	type BenchmarkingConfig = TestBenchmarkingConfig;

substrate/frame/staking/src/pallet/impls.rs

@@ -756,18 +756,32 @@ impl<T: Config> Pallet<T> {
 	/// Get the targets for an upcoming npos election.
 	///
 	/// This function is self-weighing as [`DispatchClass::Mandatory`].
-	pub fn get_npos_targets() -> Vec<T::AccountId> {
-		let mut validator_count = 0u32;
-		let targets = Validators::<T>::iter()
-			.map(|(v, _)| {
-				validator_count.saturating_inc();
-				v
-			})
-			.collect::<Vec<_>>();
+	pub fn get_npos_targets(maybe_max_len: Option<usize>) -> Vec<T::AccountId> {
+		let max_allowed_len = maybe_max_len.unwrap_or_else(|| T::TargetList::count() as usize);
+		let mut all_targets = Vec::<T::AccountId>::with_capacity(max_allowed_len);
+		let mut targets_seen = 0;
 
-		Self::register_weight(T::WeightInfo::get_npos_targets(validator_count));
+		let mut targets_iter = T::TargetList::iter();
+		while all_targets.len() < max_allowed_len &&
+			targets_seen < (NPOS_MAX_ITERATIONS_COEFFICIENT * max_allowed_len as u32)
+		{
+			let target = match targets_iter.next() {
+				Some(target) => {
+					targets_seen.saturating_inc();
+					target
+				},
+				None => break,
+			};
 
-		targets
+			if Validators::<T>::contains_key(&target) {
+				all_targets.push(target);
+			}
+		}
+
+		Self::register_weight(T::WeightInfo::get_npos_targets(all_targets.len() as u32));
+		log!(info, "generated {} npos targets", all_targets.len());
+
+		all_targets
 	}
 
 	/// This function will add a nominator to the `Nominators` storage map,
@@ -899,7 +913,7 @@ impl<T: Config> ElectionDataProvider for Pallet<T> {
 			return Err("Target snapshot too big")
 		}
 
-		Ok(Self::get_npos_targets())
+		Ok(Self::get_npos_targets(None))
 	}
 
 	fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber {
@@ -1306,6 +1320,65 @@ impl<T: Config> ScoreProvider<T::AccountId> for Pallet<T> {
 	}
 }
 
+/// A simple sorted list implementation that does not require any additional pallets. Note, this
+/// does not provide validators in sorted order. If you desire nominators in a sorted order take
+/// a look at [`pallet-bags-list`].
+pub struct UseValidatorsMap<T>(sp_std::marker::PhantomData<T>);
+impl<T: Config> SortedListProvider<T::AccountId> for UseValidatorsMap<T> {
+	type Score = BalanceOf<T>;
+	type Error = ();
+
+	/// Returns iterator over voter list, which can have `take` called on it.
+	fn iter() -> Box<dyn Iterator<Item = T::AccountId>> {
+		Box::new(Validators::<T>::iter().map(|(v, _)| v))
+	}
+	fn iter_from(
+		start: &T::AccountId,
+	) -> Result<Box<dyn Iterator<Item = T::AccountId>>, Self::Error> {
+		if Validators::<T>::contains_key(start) {
+			let start_key = Validators::<T>::hashed_key_for(start);
+			Ok(Box::new(Validators::<T>::iter_from(start_key).map(|(n, _)| n)))
+		} else {
+			Err(())
+		}
+	}
+	fn count() -> u32 {
+		Validators::<T>::count()
+	}
+	fn contains(id: &T::AccountId) -> bool {
+		Validators::<T>::contains_key(id)
+	}
+	fn on_insert(_: T::AccountId, _weight: Self::Score) -> Result<(), Self::Error> {
+		// nothing to do on insert.
+		Ok(())
+	}
+	fn get_score(id: &T::AccountId) -> Result<Self::Score, Self::Error> {
+		Ok(Pallet::<T>::weight_of(id).into())
+	}
+	fn on_update(_: &T::AccountId, _weight: Self::Score) -> Result<(), Self::Error> {
+		// nothing to do on update.
+		Ok(())
+	}
+	fn on_remove(_: &T::AccountId) -> Result<(), Self::Error> {
+		// nothing to do on remove.
+		Ok(())
+	}
+	fn unsafe_regenerate(
+		_: impl IntoIterator<Item = T::AccountId>,
+		_: Box<dyn Fn(&T::AccountId) -> Self::Score>,
+	) -> u32 {
+		// nothing to do upon regenerate.
+		0
+	}
+	fn try_state() -> Result<(), &'static str> {
+		Ok(())
+	}
+	fn unsafe_clear() {
+		#[allow(deprecated)]
+		Validators::<T>::remove_all();
+	}
+}
+
 /// A simple voter list implementation that does not require any additional pallets. Note, this
 /// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take
 /// a look at [`pallet-bags-list].
@@ -1458,6 +1531,10 @@ impl<T: Config> StakingInterface for Pallet<T> {
 #[cfg(feature = "try-runtime")]
 impl<T: Config> Pallet<T> {
 	pub(crate) fn do_try_state(_: BlockNumberFor<T>) -> Result<(), &'static str> {
+		ensure!(
+			T::VoterList::iter().all(|x| <Nominators<T>>::contains_key(&x)),
+			"VoterList contains non-nominators"
+		);
 		T::VoterList::try_state()?;
 		Self::check_nominators()?;
 		Self::check_exposures()?;

substrate/frame/staking/src/pallet/mod.rs

@@ -184,8 +184,36 @@ pub mod pallet {
 		///
 		/// The changes to nominators are reported to this. Moreover, each validator's self-vote is
 		/// also reported as one independent vote.
+		///
+		/// To keep the load off the chain as much as possible, changes made to the staked amount
+		/// via rewards and slashes are not reported and thus need to be manually fixed by the
+		/// staker. In case of `bags-list`, this always means using `rebag` and `putInFrontOf`.
+		///
+		/// Invariant: what comes out of this list will always be a nominator.
 		type VoterList: SortedListProvider<Self::AccountId, Score = VoteWeight>;
 
+		/// WIP: This is a noop as of now, the actual business logic that's described below is going
+		/// to be introduced in a follow-up PR.
+		///
+		/// Something that provides a best-effort sorted list of targets aka electable validators,
+		/// used for NPoS election.
+		///
+		/// The changes to the approval stake of each validator are reported to this. This means any
+		/// change to:
+		/// 1. The stake of any validator or nominator.
+		/// 2. The targets of any nominator
+		/// 3. The role of any staker (e.g. validator -> chilled, nominator -> validator, etc)
+		///
+		/// Unlike `VoterList`, the values in this list are always kept up to date with reward and
+		/// slash as well, and thus represent the accurate approval stake of all account being
+		/// nominated by nominators.
+		///
+		/// Note that while at the time of nomination, all targets are checked to be real
+		/// validators, they can chill at any point, and their approval stakes will still be
+		/// recorded. This implies that what comes out of iterating this list MIGHT NOT BE AN ACTIVE
+		/// VALIDATOR.
+		type TargetList: SortedListProvider<Self::AccountId, Score = BalanceOf<Self>>;
+
 		/// The maximum number of `unlocking` chunks a [`StakingLedger`] can have. Effectively
 		/// determines how many unique eras a staker may be unbonding in.
 		#[pallet::constant]

substrate/utils/frame/generate-bags/src/lib.rs

@@ -207,6 +207,10 @@ pub fn generate_thresholds<T: pallet_staking::Config>(
 	writeln!(buf, "//! Autogenerated bag thresholds.")?;
 	writeln!(buf, "//!")?;
 	writeln!(buf, "//! Generated on {}", now.to_rfc3339())?;
+	writeln!(buf, "//! Arguments")?;
+	writeln!(buf, "//! Total issuance: {}", &total_issuance)?;
+	writeln!(buf, "//! Minimum balance: {}", &minimum_balance)?;
+
 	writeln!(
 		buf,
 		"//! for the {} runtime.",
@@ -234,6 +238,17 @@ pub fn generate_thresholds<T: pallet_staking::Config>(
 	writeln!(buf)?;
 	writeln!(buf, "/// Upper thresholds delimiting the bag list.")?;
 	writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?;
+	for threshold in &thresholds {
+		num_buf.write_formatted(threshold, &format);
+		// u64::MAX, with spacers every 3 digits, is 26 characters wide
+		writeln!(buf, "	{:>26},", num_buf.as_str())?;
+	}
+	writeln!(buf, "];")?;
+
+	// thresholds balance
+	writeln!(buf)?;
+	writeln!(buf, "/// Upper thresholds delimiting the bag list.")?;
+	writeln!(buf, "pub const THRESHOLDS_BALANCES: [u128; {}] = [", thresholds.len())?;
 	for threshold in thresholds {
 		num_buf.write_formatted(&threshold, &format);
 		// u64::MAX, with spacers every 3 digits, is 26 characters wide