Reformat Validator Election (#2406)

* Add index caching to election * Initial draft of the new phragmen API. * Port post-processing to the new API. * Fix tests and accuracy. * Final fixes. * Unify convert namings. * Remove todo. * Some typos. * Bump. * Add extended balance type doc. * A bit more sophisticated weight compensation. * Fix review feedbacks. * Bump. * Final updates
2026-07-19 01:45:41 +00:00 · 2019-05-09 19:05:45 +02:00
parent c7cd82784b
commit 71426fb060
4 changed files with 477 additions and 422 deletions
@@ -16,43 +16,36 @@

 //! Rust implementation of the Phragmén election algorithm.

-use rstd::prelude::*;
-use primitives::PerU128;
-use primitives::traits::{Zero, Saturating, Convert};
-use parity_codec::{HasCompact, Encode, Decode};
-use crate::{Exposure, BalanceOf, Trait, ValidatorPrefs, IndividualExposure};
+use rstd::{prelude::*, collections::btree_map::BTreeMap};
+use primitives::{PerU128};
+use primitives::traits::{Zero, Convert, Saturating};
+use parity_codec::{Encode, Decode};
+use crate::{BalanceOf, Assignment, RawAssignment, ExpoMap, Trait, ValidatorPrefs};

 type Fraction = PerU128;
-type ExtendedBalance = u128;
-const SCALE_FACTOR_64: u128 = u64::max_value() as u128 + 1;
+/// Wrapper around the type used as the _safe_ wrapper around a `balance`.
+pub type ExtendedBalance = u128;

-/// Configure the behavior of the Phragmen election.
-/// Might be deprecated.
-pub struct ElectionConfig<Balance: HasCompact> {
-	/// Perform equalize?.
-	pub equalize: bool,
-	/// Number of equalize iterations.
-	pub iterations: usize,
-	/// Tolerance of max change per equalize iteration.
-	pub tolerance: Balance,
-}
+// this is only used while creating the candidate score. Due to reasons explained below
+// The more accurate this is, the less likely we choose a wrong candidate.
+const SCALE_FACTOR: ExtendedBalance = u32::max_value() as ExtendedBalance + 1;
+/// These are used to expose a fixed accuracy to the caller function. The bigger they are,
+/// the more accurate we get, but the more likely it is for us to overflow. The case of overflow
+/// is handled but accuracy will be lost. 32 or 16 are reasonable values.
+pub const ACCURACY: ExtendedBalance = u32::max_value() as ExtendedBalance + 1;

 /// Wrapper around validation candidates some metadata.
 #[derive(Clone, Encode, Decode, Default)]
 #[cfg_attr(feature = "std", derive(Debug))]
-pub struct Candidate<AccountId, Balance: HasCompact> {
+pub struct Candidate<AccountId> {
 	/// The validator's account
 	pub who: AccountId,
-	/// Exposure struct, holding info about the value that the validator has in stake.
-	pub exposure: Exposure<AccountId, Balance>,
 	/// Intermediary value used to sort candidates.
 	pub score: Fraction,
 	/// Accumulator of the stake of this candidate based on received votes.
 	approval_stake: ExtendedBalance,
 	/// Flag for being elected.
 	elected: bool,
-	/// This is most often equal to `Exposure.total` but not always. Needed for [`equalize`]
-	backing_stake: ExtendedBalance
 }

 /// Wrapper around the nomination info of a single nominator for a group of validators.
@@ -77,14 +70,10 @@ pub struct Edge<AccountId> {
 	who: AccountId,
 	/// Load of this vote.
 	load: Fraction,
-	/// Final backing stake of this vote.
-	backing_stake: ExtendedBalance,
-	/// Index of the candidate stored in the 'candidates' vector
+	/// Equal to `edge.load / nom.load`. Stored only to be used with post-processing.
+	ratio: ExtendedBalance,
+	/// Index of the candidate stored in the 'candidates' vector.
 	candidate_index: usize,
-	/// Index of the candidate stored in the 'elected_candidates' vector. Used only with equalize.
-	elected_idx: usize,
-	/// Indicates if this edge is a vote for an elected candidate. Used only with equalize.
-	elected: bool,
 }

 /// Perform election based on Phragmén algorithm.
@@ -93,52 +82,55 @@ pub struct Edge<AccountId> {
 ///
 /// Returns an Option of elected candidates, if election is performed.
 /// Returns None if not enough candidates exist.
+///
+/// The returned Option is a tuple consisting of:
+///   - The list of elected candidates.
+///   - The list of nominators and their associated vote weights.
 pub fn elect<T: Trait + 'static, FV, FN, FS>(
 	validator_count: usize,
 	minimum_validator_count: usize,
 	validator_iter: FV,
 	nominator_iter: FN,
 	stash_of: FS,
-	config: ElectionConfig<BalanceOf<T>>,
-) -> Option<Vec<Candidate<T::AccountId, BalanceOf<T>>>> where
+) -> Option<(Vec<T::AccountId>, Vec<(T::AccountId, Vec<RawAssignment<T>>)>)> where
 	FV: Iterator<Item=(T::AccountId, ValidatorPrefs<BalanceOf<T>>)>,
 	FN: Iterator<Item=(T::AccountId, Vec<T::AccountId>)>,
 	for <'r> FS: Fn(&'r T::AccountId) -> BalanceOf<T>,
 {
-	let into_currency = |b: BalanceOf<T>| <T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(b) as ExtendedBalance;
-	let into_votes = |b: ExtendedBalance| <T::CurrencyToVote as Convert<ExtendedBalance, BalanceOf<T>>>::convert(b);
-	let mut elected_candidates;
+	let to_votes = |b: BalanceOf<T>| <T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(b) as ExtendedBalance;
+
+	// return structures
+	let mut elected_candidates: Vec<T::AccountId>;
+	let mut assigned: Vec<(T::AccountId, Vec<RawAssignment<T>>)>;
+	let mut c_idx_cache = BTreeMap::<T::AccountId, usize>::new();

 	// 1- Pre-process candidates and place them in a container, optimisation and add phantom votes.
 	// Candidates who have 0 stake => have no votes or all null-votes. Kick them out not.
 	let mut nominators: Vec<Nominator<T::AccountId>> = Vec::with_capacity(validator_iter.size_hint().0 + nominator_iter.size_hint().0);
 	let mut candidates = validator_iter.map(|(who, _)| {
 			let stash_balance = stash_of(&who);
-			Candidate {
-				who,
-				exposure: Exposure { total: stash_balance, own: stash_balance, others: vec![] },
-				..Default::default()
-			}
+			(Candidate { who, ..Default::default() }, stash_balance)
 		})
-		.filter_map(|mut c| {
-			c.approval_stake += into_currency(c.exposure.total);
+		.filter_map(|(mut c, s)| {
+			c.approval_stake += to_votes(s);
 			if c.approval_stake.is_zero() {
 				None
 			} else {
-				Some(c)
+				Some((c, s))
 			}
 		})
 		.enumerate()
-		.map(|(idx, c)| {
+		.map(|(idx, (c, s))| {
 			nominators.push(Nominator {
 				who: c.who.clone(),
 				edges: vec![ Edge { who: c.who.clone(), candidate_index: idx, ..Default::default() }],
-				budget: into_currency(c.exposure.total),
+				budget: to_votes(s),
 				load: Fraction::zero(),
 			});
+			c_idx_cache.insert(c.who.clone(), idx);
 			c
 		})
-		.collect::<Vec<Candidate<T::AccountId, BalanceOf<T>>>>();
+		.collect::<Vec<Candidate<T::AccountId>>>();

 	// 2- Collect the nominators with the associated votes.
 	// Also collect approval stake along the way.
@@ -146,17 +138,17 @@ pub fn elect<T: Trait + 'static, FV, FN, FS>(
 		let nominator_stake = stash_of(&who);
 		let mut edges: Vec<Edge<T::AccountId>> = Vec::with_capacity(nominees.len());
 		for n in &nominees {
-			if let Some(idx) = candidates.iter_mut().position(|i| i.who == *n) {
-				candidates[idx].approval_stake = candidates[idx].approval_stake
-					.saturating_add(into_currency(nominator_stake));
-				edges.push(Edge { who: n.clone(), candidate_index: idx, ..Default::default() });
-			}
+			if let Some(idx) = c_idx_cache.get(n) {
+				// This candidate is valid + already cached.
+				candidates[*idx].approval_stake = candidates[*idx].approval_stake
+					.saturating_add(to_votes(nominator_stake));
+				edges.push(Edge { who: n.clone(), candidate_index: *idx, ..Default::default() });
+			} // else {} would be wrong votes. We don't really care about it.
 		}
-
 		Nominator {
 			who,
 			edges: edges,
-			budget: into_currency(nominator_stake),
+			budget: to_votes(nominator_stake),
 			load: Fraction::zero(),
 		}
 	}));
@@ -166,6 +158,7 @@ pub fn elect<T: Trait + 'static, FV, FN, FS>(
 		let validator_count = validator_count.min(candidates.len());

 		elected_candidates = Vec::with_capacity(validator_count);
+		assigned = Vec::with_capacity(validator_count);
 		// Main election loop
 		for _round in 0..validator_count {
 			// Loop 1: initialize score
@@ -179,12 +172,14 @@ pub fn elect<T: Trait + 'static, FV, FN, FS>(
 				for e in &n.edges {
 					let c = &mut candidates[e.candidate_index];
 					if !c.elected && !c.approval_stake.is_zero() {
+						// Basic fixed-point shifting by 32.
+						// `n.budget.saturating_mul(SCALE_FACTOR)` will never saturate
+						// since n.budget cannot exceed u64,despite being stored in u128. yet,
+						// `*n.load / SCALE_FACTOR` might collapse to zero. Hence, 32 or 16 bits are better scale factors.
+						// Note that left-associativity in operators precedence is crucially important here.
 						let temp =
-							// Basic fixed-point shifting by 64.
-							// This will never saturate since n.budget cannot exceed u64,despite being stored in u128.
-							// Note that left-associativity in operators precedence is crucially important here.
-							n.budget.saturating_mul(SCALE_FACTOR_64) / c.approval_stake
-							* (*n.load / SCALE_FACTOR_64);
+							n.budget.saturating_mul(SCALE_FACTOR) / c.approval_stake
+							* (*n.load / SCALE_FACTOR);
 						c.score = Fraction::from_max_value((*c.score).saturating_add(temp));
 					}
 				}
@@ -207,162 +202,195 @@ pub fn elect<T: Trait + 'static, FV, FN, FS>(
 					}
 				}

-				elected_candidates.push(winner.clone());
+				elected_candidates.push(winner.who.clone());
 			} else {
 				break
 			}
-		}
-		// end of all rounds
+		} // end of all rounds

 		// 4.1- Update backing stake of candidates and nominators
 		for n in &mut nominators {
+			let mut assignment = (n.who.clone(), vec![]);
 			for e in &mut n.edges {
-				// if the target of this vote is among the winners, otherwise let go.
-				if let Some(c) = elected_candidates.iter_mut().find(|c| c.who == e.who) {
-					e.elected = true;
-					e.backing_stake = n.budget.saturating_mul(*e.load) / n.load.max(1);
-					c.backing_stake = c.backing_stake.saturating_add(e.backing_stake);
-					if c.who != n.who {
-						// Only update the exposure if this vote is from some other account.
-						c.exposure.total = c.exposure.total.saturating_add(into_votes(e.backing_stake));
-						c.exposure.others.push(
-							IndividualExposure { who: n.who.clone(), value: into_votes(e.backing_stake) }
-						);
+				if let Some(c) = elected_candidates.iter().find(|c| **c == e.who) {
+					if *c != n.who {
+						let ratio = {
+							// Full support. No need to calculate.
+							if *n.load == *e.load { ACCURACY }
+							else {
+								// This should not saturate. Safest is to just check
+								if let Some(r) = ACCURACY.checked_mul(*e.load) {
+									r / n.load.max(1)
+								} else {
+									// Just a simple trick.
+									*e.load / (n.load.max(1) / ACCURACY)
+								}
+							}
+						};
+						e.ratio = ratio;
+						assignment.1.push((e.who.clone(), ratio));
 					}
 				}
 			}
-		}

-		// Optionally perform equalize post-processing.
-		if config.equalize {
-			let tolerance = config.tolerance;
-			let equalize_iterations = config.iterations;
+			if assignment.1.len() > 0 {
+				// To ensure an assertion indicating: no stake from the nominator going to waste,
+				// we add a minimal post-processing to equally assign all of the leftover stake ratios.
+				let vote_count = assignment.1.len() as ExtendedBalance;
+				let l = assignment.1.len();
+				let sum = assignment.1.iter().map(|a| a.1).sum::<ExtendedBalance>();
+				let diff = ACCURACY.checked_sub(sum).unwrap_or(0);
+				let diff_per_vote= diff / vote_count;

-			// Fix indexes
-			nominators.iter_mut().for_each(|n| {
-				n.edges.iter_mut().for_each(|e| {
-					if let Some(idx) = elected_candidates.iter().position(|c| c.who == e.who) {
-						e.elected_idx = idx;
+				if diff_per_vote > 0 {
+					for i in 0..l {
+						assignment.1[i%l].1 =
+							assignment.1[i%l].1
+							.saturating_add(diff_per_vote);
 					}
-				});
-			});
-
-			for _i in 0..equalize_iterations {
-				let mut max_diff = <BalanceOf<T>>::zero();
-				nominators.iter_mut().for_each(|mut n| {
-					let diff = equalize::<T>(&mut n, &mut elected_candidates, tolerance);
-					if diff > max_diff {
-						max_diff = diff;
-					}
-				});
-				if max_diff < tolerance {
-					break;
 				}
+
+				// `remainder` is set to be less than maximum votes of a nominator (currently 16).
+				// safe to cast it to usize.
+				let remainder = diff - diff_per_vote * vote_count;
+				for i in 0..remainder as usize {
+					assignment.1[i%l].1 =
+						assignment.1[i%l].1
+						.saturating_add(1);
+				}
+				assigned.push(assignment);
 			}
 		}
+
 	} else {
 		// if we have less than minimum, use the previous validator set.
 		return None
 	}
-	Some(elected_candidates)
+	Some((elected_candidates, assigned))
 }

 /// Performs equalize post-processing to the output of the election algorithm
 /// This function mutates the input parameters, most noticeably it updates the exposure of
 /// the elected candidates.
-/// The return value is to tolerance at which the function has stopped.
+///
+/// No value is returned from the function and the `expo_map` parameter is updated.
 pub fn equalize<T: Trait + 'static>(
-	nominator: &mut Nominator<T::AccountId>,
-	elected_candidates: &mut Vec<Candidate<T::AccountId, BalanceOf<T>>>,
-	_tolerance: BalanceOf<T>
-) -> BalanceOf<T> {
-	let into_currency = |b: BalanceOf<T>|    <T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(b) as ExtendedBalance;
-	let into_votes = |b: ExtendedBalance| <T::CurrencyToVote as Convert<ExtendedBalance, BalanceOf<T>>>::convert(b);
-	let tolerance = into_currency(_tolerance);
-
-	let mut elected_edges = nominator.edges
-		.iter_mut()
-		.filter(|e| e.elected)
-		.collect::<Vec<&mut Edge<T::AccountId>>>();
-
-	if elected_edges.len() == 0 {
-		return <BalanceOf<T>>::zero();
+	assignments: &mut Vec<(T::AccountId, BalanceOf<T>, Vec<Assignment<T>>)>,
+	expo_map: &mut ExpoMap<T>,
+	tolerance: ExtendedBalance,
+	iterations: usize,
+) {
+	for _i in 0..iterations {
+		let mut max_diff = 0;
+		assignments.iter_mut().for_each(|(n, budget, assignment)| {
+			let diff = do_equalize::<T>(&n, *budget, assignment, expo_map, tolerance);
+			if diff > max_diff {
+				max_diff = diff;
+			}
+		});
+		if max_diff < tolerance {
+			break;
+		}
 	}
+}
+
+fn do_equalize<T: Trait + 'static>(
+	nominator: &T::AccountId,
+	budget_balance: BalanceOf<T>,
+	elected_edges_balance: &mut Vec<Assignment<T>>,
+	expo_map: &mut ExpoMap<T>,
+	tolerance: ExtendedBalance
+) -> ExtendedBalance {
+	let to_votes = |b: BalanceOf<T>| <T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(b) as ExtendedBalance;
+	let to_balance = |v: ExtendedBalance| <T::CurrencyToVote as Convert<ExtendedBalance, BalanceOf<T>>>::convert(v);
+	let budget = to_votes(budget_balance);
+
+	// Convert all stakes to extended. Result is Vec<(Acc, Ratio, Balance)>
+	let mut elected_edges = elected_edges_balance
+		.into_iter()
+		.map(|e| (e.0.clone(), e.1, to_votes(e.2)))
+		.collect::<Vec<(T::AccountId, ExtendedBalance, ExtendedBalance)>>();

 	let stake_used = elected_edges
 		.iter()
-		.fold(0, |s, e| s.saturating_add(e.backing_stake));
-	let backed_stakes = elected_edges
+		.fold(0 as ExtendedBalance, |s, e| s.saturating_add(e.2));
+
+	let backed_stakes_iter = elected_edges
 		.iter()
-		.map(|e| elected_candidates[e.elected_idx].backing_stake)
-		.collect::<Vec<ExtendedBalance>>();
+		.filter_map(|e| expo_map.get(&e.0))
+		.map(|e| to_votes(e.total));
+
 	let backing_backed_stake = elected_edges
 		.iter()
-		.filter(|e| e.backing_stake > 0)
-		.map(|e| elected_candidates[e.elected_idx].backing_stake)
+		.filter(|e| e.2 > 0)
+		.filter_map(|e| expo_map.get(&e.0))
+		.map(|e| to_votes(e.total))
 		.collect::<Vec<ExtendedBalance>>();

 	let mut difference;
 	if backing_backed_stake.len() > 0 {
-		let max_stake = *backing_backed_stake
+		let max_stake = backing_backed_stake
 			.iter()
 			.max()
 			.expect("vector with positive length will have a max; qed");
-		let min_stake = *backed_stakes
-			.iter()
+		let min_stake = backed_stakes_iter
 			.min()
-			.expect("vector with positive length will have a min; qed");
+			.expect("iterator with positive length will have a min; qed");
+
 		difference = max_stake.saturating_sub(min_stake);
-		difference = difference.saturating_add(nominator.budget.saturating_sub(stake_used));
+		difference = difference.saturating_add(budget.saturating_sub(stake_used));
 		if difference < tolerance {
-			return into_votes(difference);
+			return difference;
 		}
 	} else {
-		difference = nominator.budget;
+		difference = budget;
 	}

 	// Undo updates to exposure
 	elected_edges.iter_mut().for_each(|e| {
-		// NOTE: no assertions in the runtime, but this should nonetheless be indicative.
-		//assert_eq!(elected_candidates[e.elected_idx].who, e.who);
-		elected_candidates[e.elected_idx].backing_stake -= e.backing_stake;
-		elected_candidates[e.elected_idx].exposure.total -= into_votes(e.backing_stake);
-		e.backing_stake = 0;
+		if let Some(expo) = expo_map.get_mut(&e.0) {
+			expo.total = expo.total.saturating_sub(to_balance(e.2));
+		}
+		e.2 = 0;
 	});

-	elected_edges.sort_unstable_by_key(|e| elected_candidates[e.elected_idx].backing_stake);
+	elected_edges.sort_unstable_by_key(|e| e.2);

 	let mut cumulative_stake: ExtendedBalance = 0;
 	let mut last_index = elected_edges.len() - 1;
-	let budget = nominator.budget;
 	elected_edges.iter_mut().enumerate().for_each(|(idx, e)| {
-		let stake = elected_candidates[e.elected_idx].backing_stake;
+		if let Some(expo) = expo_map.get_mut(&e.0) {
+			let stake: ExtendedBalance = to_votes(expo.total);

-		let stake_mul = stake.saturating_mul(idx as ExtendedBalance);
-		let stake_sub = stake_mul.saturating_sub(cumulative_stake);
-		if stake_sub > budget {
-			last_index = idx.checked_sub(1).unwrap_or(0);
-			return
+			let stake_mul = stake.saturating_mul(idx as ExtendedBalance);
+			let stake_sub = stake_mul.saturating_sub(cumulative_stake);
+			if stake_sub > budget {
+				last_index = idx.checked_sub(1).unwrap_or(0);
+				return
+			}
+			cumulative_stake = cumulative_stake.saturating_add(stake);
 		}
-		cumulative_stake = cumulative_stake.saturating_add(stake);
 	});

-	let last_stake = elected_candidates[elected_edges[last_index].elected_idx].backing_stake;
+	let last_stake = elected_edges[last_index].2;
 	let split_ways = last_index + 1;
-	let excess = nominator.budget
+	let excess = budget
 		.saturating_add(cumulative_stake)
 		.saturating_sub(last_stake.saturating_mul(split_ways as ExtendedBalance));
-	let nominator_address = nominator.who.clone();
 	elected_edges.iter_mut().take(split_ways).for_each(|e| {
-		let c = &mut elected_candidates[e.elected_idx];
-		e.backing_stake = (excess / split_ways as ExtendedBalance)
-			.saturating_add(last_stake)
-			.saturating_sub(c.backing_stake);
-		c.exposure.total = c.exposure.total.saturating_add(into_votes(e.backing_stake));
-		c.backing_stake = c.backing_stake.saturating_add(e.backing_stake);
-		if let Some(i_expo) = c.exposure.others.iter_mut().find(|i| i.who == nominator_address) {
-			i_expo.value = into_votes(e.backing_stake);
+		if let Some(expo) = expo_map.get_mut(&e.0) {
+			e.2 = (excess / split_ways as ExtendedBalance)
+				.saturating_add(last_stake)
+				.saturating_sub(to_votes(expo.total));
+			expo.total = expo.total.saturating_add(to_balance(e.2));
+			if let Some(i_expo) = expo.others.iter_mut().find(|i| i.who == nominator.clone()) {
+				i_expo.value = to_balance(e.2);
+			}
 		}
 	});
-	into_votes(difference)
+
+	// Store back the individual edge weights.
+	elected_edges.iter().enumerate().for_each(|(idx, e)| elected_edges_balance[idx].2 = to_balance(e.2));
+
+	difference
 }