[Feature] Part 1: add TargetList for validator ranking (#12034)

* [Feature] Part 1: add TargetList for validator ranking

* remove redundant todo

* remove typo

* cleanup

* implement score

* more fixes

* fix thresholds

* fmt

* Remove the stuff that has to come in the next PR, some fixes

* extended balance import

* Change all the references from VoteWeight to Self::Score

* Add a migration for VoterBagsList

* fix score

* add targetList to nomination-pools tests

* fix bench

* address review comments

* change get_npos_targets

* address more comments

* remove thresholds for the time being

* fix instance reference

* VoterBagsListInstance

* reus

* remove params that are not used yet

* Introduced pre/post upgrade try-runtime checks

* fix

* fixes

* fix migration

* fix migration

* fix post_upgrade

* change

* Fix

* eloquent PhantomData

* fix PD

* more fixes

* Update frame/staking/src/pallet/impls.rs

Co-authored-by: Squirrel <gilescope@gmail.com>

* is_nominator now works

* fix test-staking

* build fixes

* fix remote-tests

* Apply suggestions from code review

Co-authored-by: parity-processbot <>
Co-authored-by: kianenigma <kian@parity.io>
Co-authored-by: Squirrel <gilescope@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

substrate/frame/staking/src/benchmarking.rs

@@ -844,7 +844,7 @@ benchmarks! {
 			v, n, T::MaxNominations::get() as usize, false, None
 		)?;
 	}: {
-		let targets = <Staking<T>>::get_npos_targets();
+		let targets = <Staking<T>>::get_npos_targets(None);
 		assert_eq!(targets.len() as u32, v);
 	}
 

substrate/frame/staking/src/lib.rs

@@ -886,11 +886,12 @@ enum Releases {
 	V8_0_0,  // populate `VoterList`.
 	V9_0_0,  // inject validators into `VoterList` as well.
 	V10_0_0, // remove `EarliestUnappliedSlash`.
+	V11_0_0, // Move pallet storage prefix, e.g. BagsList -> VoterBagsList
 }
 
 impl Default for Releases {
 	fn default() -> Self {
-		Releases::V10_0_0
+		Releases::V11_0_0
 	}
 }
 

substrate/frame/staking/src/migrations.rs

@@ -20,6 +20,99 @@ use super::*;
 use frame_election_provider_support::SortedListProvider;
 use frame_support::traits::OnRuntimeUpgrade;
 
+pub mod v11 {
+	use super::*;
+	use frame_support::{
+		storage::migration::move_pallet,
+		traits::{GetStorageVersion, PalletInfoAccess},
+	};
+	#[cfg(feature = "try-runtime")]
+	use sp_io::hashing::twox_128;
+
+	pub struct MigrateToV11<T, P, N>(sp_std::marker::PhantomData<(T, P, N)>);
+	impl<T: Config, P: GetStorageVersion + PalletInfoAccess, N: Get<&'static str>> OnRuntimeUpgrade
+		for MigrateToV11<T, P, N>
+	{
+		#[cfg(feature = "try-runtime")]
+		fn pre_upgrade() -> Result<(), &'static str> {
+			frame_support::ensure!(
+				StorageVersion::<T>::get() == crate::Releases::V10_0_0,
+				"must upgrade linearly"
+			);
+			let old_pallet_prefix = twox_128(N::get().as_bytes());
+
+			frame_support::ensure!(
+				sp_io::storage::next_key(&old_pallet_prefix).is_some(),
+				"no data for the old pallet name has been detected"
+			);
+
+			Ok(())
+		}
+
+		/// Migrate the entire storage of this pallet to a new prefix.
+		///
+		/// This new prefix must be the same as the one set in construct_runtime. For safety, use
+		/// `PalletInfo` to get it, as:
+		/// `<Runtime as frame_system::Config>::PalletInfo::name::<VoterBagsList>`.
+		///
+		/// The migration will look into the storage version in order to avoid triggering a
+		/// migration on an up to date storage.
+		fn on_runtime_upgrade() -> Weight {
+			let old_pallet_name = N::get();
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+
+			if StorageVersion::<T>::get() == Releases::V10_0_0 {
+				// bump version anyway, even if we don't need to move the prefix
+				StorageVersion::<T>::put(Releases::V11_0_0);
+				if new_pallet_name == old_pallet_name {
+					log!(
+						warn,
+						"new bags-list name is equal to the old one, only bumping the version"
+					);
+					return T::DbWeight::get().reads(1).saturating_add(T::DbWeight::get().writes(1))
+				}
+
+				move_pallet(old_pallet_name.as_bytes(), new_pallet_name.as_bytes());
+				<T as frame_system::Config>::BlockWeights::get().max_block
+			} else {
+				log!(warn, "v11::migrate should be removed.");
+				T::DbWeight::get().reads(1)
+			}
+		}
+
+		#[cfg(feature = "try-runtime")]
+		fn post_upgrade() -> Result<(), &'static str> {
+			frame_support::ensure!(
+				StorageVersion::<T>::get() == crate::Releases::V11_0_0,
+				"wrong version after the upgrade"
+			);
+
+			let old_pallet_name = N::get();
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+
+			// skip storage prefix checks for the same pallet names
+			if new_pallet_name == old_pallet_name {
+				return Ok(())
+			}
+
+			let old_pallet_prefix = twox_128(N::get().as_bytes());
+			frame_support::ensure!(
+				sp_io::storage::next_key(&old_pallet_prefix).is_none(),
+				"old pallet data hasn't been removed"
+			);
+
+			let new_pallet_name = <P as PalletInfoAccess>::name();
+			let new_pallet_prefix = twox_128(new_pallet_name.as_bytes());
+			frame_support::ensure!(
+				sp_io::storage::next_key(&new_pallet_prefix).is_some(),
+				"new pallet data hasn't been created"
+			);
+
+			Ok(())
+		}
+	}
+}
+
 pub mod v10 {
 	use super::*;
 	use frame_support::storage_alias;

substrate/frame/staking/src/mock.rs

@@ -99,7 +99,7 @@ frame_support::construct_runtime!(
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
 		Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>},
 		Historical: pallet_session::historical::{Pallet, Storage},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterBagsList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 	}
 );
 
@@ -240,9 +240,11 @@ parameter_types! {
 	pub static LedgerSlashPerEra: (BalanceOf<Test>, BTreeMap<EraIndex, BalanceOf<Test>>) = (Zero::zero(), BTreeMap::new());
 }
 
-impl pallet_bags_list::Config for Test {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Test {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
+	// Staking is the source of truth for voter bags list, since they are not kept up to date.
 	type ScoreProvider = Staking;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
@@ -296,7 +298,8 @@ impl crate::pallet::pallet::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	// NOTE: consider a macro and use `UseNominatorsAndValidatorsMap<Self>` as well.
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	type TargetList = UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = OnStakerSlashMock<Test>;
 	type BenchmarkingConfig = TestBenchmarkingConfig;

substrate/frame/staking/src/pallet/impls.rs

@@ -756,18 +756,32 @@ impl<T: Config> Pallet<T> {
 	/// Get the targets for an upcoming npos election.
 	///
 	/// This function is self-weighing as [`DispatchClass::Mandatory`].
-	pub fn get_npos_targets() -> Vec<T::AccountId> {
-		let mut validator_count = 0u32;
-		let targets = Validators::<T>::iter()
-			.map(|(v, _)| {
-				validator_count.saturating_inc();
-				v
-			})
-			.collect::<Vec<_>>();
+	pub fn get_npos_targets(maybe_max_len: Option<usize>) -> Vec<T::AccountId> {
+		let max_allowed_len = maybe_max_len.unwrap_or_else(|| T::TargetList::count() as usize);
+		let mut all_targets = Vec::<T::AccountId>::with_capacity(max_allowed_len);
+		let mut targets_seen = 0;
 
-		Self::register_weight(T::WeightInfo::get_npos_targets(validator_count));
+		let mut targets_iter = T::TargetList::iter();
+		while all_targets.len() < max_allowed_len &&
+			targets_seen < (NPOS_MAX_ITERATIONS_COEFFICIENT * max_allowed_len as u32)
+		{
+			let target = match targets_iter.next() {
+				Some(target) => {
+					targets_seen.saturating_inc();
+					target
+				},
+				None => break,
+			};
 
-		targets
+			if Validators::<T>::contains_key(&target) {
+				all_targets.push(target);
+			}
+		}
+
+		Self::register_weight(T::WeightInfo::get_npos_targets(all_targets.len() as u32));
+		log!(info, "generated {} npos targets", all_targets.len());
+
+		all_targets
 	}
 
 	/// This function will add a nominator to the `Nominators` storage map,
@@ -899,7 +913,7 @@ impl<T: Config> ElectionDataProvider for Pallet<T> {
 			return Err("Target snapshot too big")
 		}
 
-		Ok(Self::get_npos_targets())
+		Ok(Self::get_npos_targets(None))
 	}
 
 	fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber {
@@ -1306,6 +1320,65 @@ impl<T: Config> ScoreProvider<T::AccountId> for Pallet<T> {
 	}
 }
 
+/// A simple sorted list implementation that does not require any additional pallets. Note, this
+/// does not provide validators in sorted order. If you desire nominators in a sorted order take
+/// a look at [`pallet-bags-list`].
+pub struct UseValidatorsMap<T>(sp_std::marker::PhantomData<T>);
+impl<T: Config> SortedListProvider<T::AccountId> for UseValidatorsMap<T> {
+	type Score = BalanceOf<T>;
+	type Error = ();
+
+	/// Returns iterator over voter list, which can have `take` called on it.
+	fn iter() -> Box<dyn Iterator<Item = T::AccountId>> {
+		Box::new(Validators::<T>::iter().map(|(v, _)| v))
+	}
+	fn iter_from(
+		start: &T::AccountId,
+	) -> Result<Box<dyn Iterator<Item = T::AccountId>>, Self::Error> {
+		if Validators::<T>::contains_key(start) {
+			let start_key = Validators::<T>::hashed_key_for(start);
+			Ok(Box::new(Validators::<T>::iter_from(start_key).map(|(n, _)| n)))
+		} else {
+			Err(())
+		}
+	}
+	fn count() -> u32 {
+		Validators::<T>::count()
+	}
+	fn contains(id: &T::AccountId) -> bool {
+		Validators::<T>::contains_key(id)
+	}
+	fn on_insert(_: T::AccountId, _weight: Self::Score) -> Result<(), Self::Error> {
+		// nothing to do on insert.
+		Ok(())
+	}
+	fn get_score(id: &T::AccountId) -> Result<Self::Score, Self::Error> {
+		Ok(Pallet::<T>::weight_of(id).into())
+	}
+	fn on_update(_: &T::AccountId, _weight: Self::Score) -> Result<(), Self::Error> {
+		// nothing to do on update.
+		Ok(())
+	}
+	fn on_remove(_: &T::AccountId) -> Result<(), Self::Error> {
+		// nothing to do on remove.
+		Ok(())
+	}
+	fn unsafe_regenerate(
+		_: impl IntoIterator<Item = T::AccountId>,
+		_: Box<dyn Fn(&T::AccountId) -> Self::Score>,
+	) -> u32 {
+		// nothing to do upon regenerate.
+		0
+	}
+	fn try_state() -> Result<(), &'static str> {
+		Ok(())
+	}
+	fn unsafe_clear() {
+		#[allow(deprecated)]
+		Validators::<T>::remove_all();
+	}
+}
+
 /// A simple voter list implementation that does not require any additional pallets. Note, this
 /// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take
 /// a look at [`pallet-bags-list].
@@ -1458,6 +1531,10 @@ impl<T: Config> StakingInterface for Pallet<T> {
 #[cfg(feature = "try-runtime")]
 impl<T: Config> Pallet<T> {
 	pub(crate) fn do_try_state(_: BlockNumberFor<T>) -> Result<(), &'static str> {
+		ensure!(
+			T::VoterList::iter().all(|x| <Nominators<T>>::contains_key(&x)),
+			"VoterList contains non-nominators"
+		);
 		T::VoterList::try_state()?;
 		Self::check_nominators()?;
 		Self::check_exposures()?;

substrate/frame/staking/src/pallet/mod.rs

@@ -184,8 +184,36 @@ pub mod pallet {
 		///
 		/// The changes to nominators are reported to this. Moreover, each validator's self-vote is
 		/// also reported as one independent vote.
+		///
+		/// To keep the load off the chain as much as possible, changes made to the staked amount
+		/// via rewards and slashes are not reported and thus need to be manually fixed by the
+		/// staker. In case of `bags-list`, this always means using `rebag` and `putInFrontOf`.
+		///
+		/// Invariant: what comes out of this list will always be a nominator.
 		type VoterList: SortedListProvider<Self::AccountId, Score = VoteWeight>;
 
+		/// WIP: This is a noop as of now, the actual business logic that's described below is going
+		/// to be introduced in a follow-up PR.
+		///
+		/// Something that provides a best-effort sorted list of targets aka electable validators,
+		/// used for NPoS election.
+		///
+		/// The changes to the approval stake of each validator are reported to this. This means any
+		/// change to:
+		/// 1. The stake of any validator or nominator.
+		/// 2. The targets of any nominator
+		/// 3. The role of any staker (e.g. validator -> chilled, nominator -> validator, etc)
+		///
+		/// Unlike `VoterList`, the values in this list are always kept up to date with reward and
+		/// slash as well, and thus represent the accurate approval stake of all account being
+		/// nominated by nominators.
+		///
+		/// Note that while at the time of nomination, all targets are checked to be real
+		/// validators, they can chill at any point, and their approval stakes will still be
+		/// recorded. This implies that what comes out of iterating this list MIGHT NOT BE AN ACTIVE
+		/// VALIDATOR.
+		type TargetList: SortedListProvider<Self::AccountId, Score = BalanceOf<Self>>;
+
 		/// The maximum number of `unlocking` chunks a [`StakingLedger`] can have. Effectively
 		/// determines how many unique eras a staker may be unbonding in.
 		#[pallet::constant]