Offchain Phragmén BREAKING. (#4517)

* Initial skeleton for offchain phragmen

* Basic compact encoding decoding for results

* add compact files

* Bring back Self::ensure_storage_upgraded();

* Make staking use compact stuff.

* First seemingly working version of reduce, full of todos

* Everything phragmen related works again.

* Signing made easier, still issues.

* Signing from offchain compile fine 😎

* make compact work with staked asssignment

* Evaluation basics are in place.

* Move reduce into crate. Document stuff

* move reduce into no_std

* Add files

* Remove other std deps. Runtime compiles

* Seemingly it is al stable; cycle implemented but not integrated.

* Add fuzzing code.

* Cleanup reduce a bit more.

* a metric ton of tests for staking; wip 🔨

* Implement a lot more of the tests.

* wip getting the unsigned stuff to work

* A bit gleanup for unsigned debug

* Clean and finalize compact code.

* Document reduce.

* Still problems with signing

* We officaly duct taped the transaction submission stuff. 🤓

* Deadlock with keys again

* Runtime builds

* Unsigned test works 🙌

* Some cleanups

* Make all the tests compile and stuff

* Minor cleanup

* fix more merge stuff

* Most tests work again.

* a very nasty bug in reduce

* Fix all integrations

* Fix more todos

* Revamp everything and everything

* Remove bogus test

* Some review grumbles.

* Some fixes

* Fix doc test

* loop for submission

* Fix cli, keyring etc.

* some cleanup

* Fix staking tests again

* fix per-things; bring patches from benchmarking

* better score prediction

* Add fuzzer, more patches.

* Some fixes

* More docs

* Remove unused generics

* Remove max-nominator footgun

* Better fuzzer

* Disable it ❌

* Bump.

* Another round of self-review

* Refactor a lot

* More major fixes in perThing

* Add new fuzz file

* Update lock

* fix fuzzing code.

* Fix nominator retain test

* Add slashing check

* Update frame/staking/src/tests.rs

Co-Authored-By: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>

* Some formatting nits

* Review comments.

* Fix cargo file

* Almost all tests work again

* Update frame/staking/src/tests.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Fix review comments

* More review stuff

* Some nits

* Fix new staking / session / babe relation

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Update primitives/phragmen/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Update primitives/phragmen/compact/src/lib.rs

Co-Authored-By: thiolliere <gui.thiolliere@gmail.com>

* Some doc updates to slashing

* Fix derive

* Remove imports

* Remove unimplemented tests

* nits

* Remove dbg

* Better fuzzing params

* Remove unused pref map

* Deferred Slashing/Offence for offchain Phragmen  (#5151)

* Some boilerplate

* Add test

* One more test

* Review comments

* Fix build

* review comments

* fix more

* fix build

* Some cleanups and self-reviews

* More minor self reviews

* Final nits

* Some merge fixes.

* opt comment

* Fix build

* Fix build again.

* Update frame/staking/fuzz/fuzz_targets/submit_solution.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Update frame/staking/src/slashing.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Update frame/staking/src/offchain_election.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

* Fix review comments

* fix test

* === 🔑 Revamp without staking key.

* final round of changes.

* Fix cargo-deny

* Update frame/staking/src/lib.rs

Co-Authored-By: Gavin Wood <gavin@parity.io>

Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Co-authored-by: thiolliere <gui.thiolliere@gmail.com>
Co-authored-by: Gavin Wood <gavin@parity.io>

substrate/frame/authority-discovery/src/lib.rs

@@ -121,6 +121,7 @@ mod tests {
 		type ValidatorId = AuthorityId;
 		type ValidatorIdOf = ConvertInto;
 		type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+		type NextSessionRotation = pallet_session::PeriodicSessions<Period, Offset>;
 	}
 
 	impl pallet_session::historical::Trait for Test {

substrate/frame/babe/Cargo.toml

@@ -9,7 +9,6 @@ repository = "https://github.com/paritytech/substrate/"
 description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions."
 
 [dependencies]
-hex-literal = "0.2.1"
 codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.101", optional = true }
 sp-inherents = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/inherents" }
@@ -26,11 +25,7 @@ sp-consensus-vrf = { version = "0.8.0-alpha.5", default-features = false, path =
 sp-io = { path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"}
 
 [dev-dependencies]
-lazy_static = "1.4.0"
-parking_lot = "0.10.0"
-sp-version = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/version" }
 sp-core = { version = "2.0.0-alpha.5", path = "../../primitives/core" }
-substrate-test-runtime = { version = "2.0.0-dev", path = "../../test-utils/runtime" }
 
 [features]
 default = ["std"]

substrate/frame/babe/src/lib.rs

@@ -19,8 +19,8 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 #![forbid(unused_must_use, unsafe_code, unused_variables, unused_must_use)]
-#![deny(unused_imports)]
-pub use pallet_timestamp;
+
+use pallet_timestamp;
 
 use sp_std::{result, prelude::*};
 use frame_support::{
@@ -29,7 +29,7 @@ use frame_support::{
 };
 use sp_timestamp::OnTimestampSet;
 use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill, PerThing};
-use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash};
+use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash, One};
 use sp_staking::{
 	SessionIndex,
 	offence::{Offence, Kind},
@@ -307,12 +307,34 @@ impl<T: Trait> Module<T> {
 		// epoch 0 as having started at the slot of block 1. We want to use
 		// the same randomness and validator set as signalled in the genesis,
 		// so we don't rotate the epoch.
-		now != sp_runtime::traits::One::one() && {
+		now != One::one() && {
 			let diff = CurrentSlot::get().saturating_sub(Self::current_epoch_start());
 			diff >= T::EpochDuration::get()
 		}
 	}
 
+	/// Return the _best guess_ block number, at which the next epoch change is predicted to happen.
+	///
+	/// Returns None if the prediction is in the past; This implies an error internally in the Babe
+	/// and should not happen under normal circumstances.
+	///
+	/// In other word, this is only accurate if no slots are missed. Given missed slots, the slot
+	/// number will grow while the block number will not. Hence, the result can be interpreted as an
+	/// upper bound.
+	// -------------- IMPORTANT NOTE --------------
+	// This implementation is linked to how [`should_epoch_change`] is working. This might need to
+	// be updated accordingly, if the underlying mechanics of slot and epochs change.
+	pub fn next_expected_epoch_change(now: T::BlockNumber) -> Option<T::BlockNumber> {
+		let next_slot = Self::current_epoch_start().saturating_add(T::EpochDuration::get());
+		next_slot
+			.checked_sub(CurrentSlot::get())
+			.map(|slots_remaining| {
+				// This is a best effort guess. Drifts in the slot/block ratio will cause errors here.
+				let blocks_remaining: T::BlockNumber = slots_remaining.saturated_into();
+				now.saturating_add(blocks_remaining)
+			})
+	}
+
 	/// DANGEROUS: Enact an epoch change. Should be done on every block where `should_epoch_change` has returned `true`,
 	/// and the caller is the only caller of this function.
 	///
@@ -324,10 +346,7 @@ impl<T: Trait> Module<T> {
 	) {
 		// PRECONDITION: caller has done initialization and is guaranteed
 		// by the session module to be called before this.
-		#[cfg(debug_assertions)]
-		{
-			assert!(Self::initialized().is_some())
-		}
+		debug_assert!(Self::initialized().is_some());
 
 		// Update epoch index
 		let epoch_index = EpochIndex::get()
@@ -473,6 +492,12 @@ impl<T: Trait> OnTimestampSet<T::Moment> for Module<T> {
 	fn on_timestamp_set(_moment: T::Moment) { }
 }
 
+impl<T: Trait> frame_support::traits::EstimateNextSessionRotation<T::BlockNumber> for Module<T> {
+	fn estimate_next_session_rotation(now: T::BlockNumber) -> Option<T::BlockNumber> {
+		Self::next_expected_epoch_change(now)
+	}
+}
+
 impl<T: Trait> sp_runtime::BoundToRuntimeAppPublic for Module<T> {
 	type Public = AuthorityId;
 }

substrate/frame/babe/src/mock.rs

@@ -16,14 +16,22 @@
 
 //! Test utilities
 
-use super::{Trait, Module, GenesisConfig};
+use codec::Encode;
+use super::{Trait, Module, GenesisConfig, CurrentSlot};
 use sp_runtime::{
-	traits::IdentityLookup, Perbill, testing::{Header, UintAuthorityId}, impl_opaque_keys,
+	Perbill, impl_opaque_keys,
+	testing::{Header, UintAuthorityId, Digest, DigestItem},
+	traits::IdentityLookup,
+};
+use frame_system::InitKind;
+use frame_support::{
+	impl_outer_origin, parameter_types, StorageValue,
+	traits::OnInitialize,
+	weights::Weight,
 };
-use sp_version::RuntimeVersion;
-use frame_support::{impl_outer_origin, parameter_types, weights::Weight};
 use sp_io;
 use sp_core::H256;
+use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof};
 
 impl_outer_origin!{
 	pub enum Origin for Test  where system = frame_system {}
@@ -43,7 +51,6 @@ parameter_types! {
 	pub const MinimumPeriod: u64 = 1;
 	pub const EpochDuration: u64 = 3;
 	pub const ExpectedBlockTime: u64 = 1;
-	pub const Version: RuntimeVersion = substrate_test_runtime::VERSION;
 	pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(16);
 }
 
@@ -53,7 +60,7 @@ impl frame_system::Trait for Test {
 	type BlockNumber = u64;
 	type Call = ();
 	type Hash = H256;
-	type Version = Version;
+	type Version = ();
 	type Hashing = sp_runtime::traits::BlakeTwo256;
 	type AccountId = DummyValidatorId;
 	type Lookup = IdentityLookup<Self::AccountId>;
@@ -79,11 +86,12 @@ impl pallet_session::Trait for Test {
 	type Event = ();
 	type ValidatorId = <Self as frame_system::Trait>::AccountId;
 	type ShouldEndSession = Babe;
-	type SessionHandler = (Babe,Babe,);
+	type SessionHandler = (Babe,);
 	type SessionManager = ();
 	type ValidatorIdOf = ();
 	type Keys = MockSessionKeys;
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+	type NextSessionRotation = Babe;
 }
 
 impl pallet_timestamp::Trait for Test {
@@ -106,5 +114,44 @@ pub fn new_test_ext(authorities: Vec<DummyValidatorId>) -> sp_io::TestExternalit
 	t.into()
 }
 
+pub fn go_to_block(n: u64, s: u64) {
+	let pre_digest = make_pre_digest(0, s, RawVRFOutput([1; 32]), RawVRFProof([0xff; 64]));
+	System::initialize(&n, &Default::default(), &Default::default(), &pre_digest, InitKind::Full);
+	System::set_block_number(n);
+	if s > 1 {
+		CurrentSlot::put(s);
+	}
+	// includes a call into `Babe::do_initialize`.
+	Session::on_initialize(n);
+}
+
+/// Slots will grow accordingly to blocks
+pub fn progress_to_block(n: u64) {
+	let mut slot = Babe::current_slot() + 1;
+	for i in System::block_number()+1..=n {
+		go_to_block(i, slot);
+		slot += 1;
+	}
+}
+
+pub fn make_pre_digest(
+	authority_index: sp_consensus_babe::AuthorityIndex,
+	slot_number: sp_consensus_babe::SlotNumber,
+	vrf_output: RawVRFOutput,
+	vrf_proof: RawVRFProof,
+) -> Digest {
+	let digest_data = sp_consensus_babe::digests::RawPreDigest::Primary(
+		sp_consensus_babe::digests::RawPrimaryPreDigest {
+			authority_index,
+			slot_number,
+			vrf_output,
+			vrf_proof,
+		}
+	);
+	let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode());
+	Digest { logs: vec![log] }
+}
+
 pub type System = frame_system::Module<Test>;
 pub type Babe = Module<Test>;
+pub type Session = pallet_session::Module<Test>;

substrate/frame/babe/src/tests.rs

@@ -17,11 +17,10 @@
 //! Consensus extension module tests for BABE consensus.
 
 use super::*;
+use mock::*;
 use frame_support::traits::OnFinalize;
-use mock::{new_test_ext, Babe, System};
-use sp_runtime::testing::{Digest, DigestItem};
-use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof};
 use pallet_session::ShouldEndSession;
+use sp_consensus_vrf::schnorrkel::{RawVRFOutput, RawVRFProof};
 
 const EMPTY_RANDOMNESS: [u8; 32] = [
 	74, 25, 49, 128, 53, 97, 244, 49,
@@ -30,24 +29,6 @@ const EMPTY_RANDOMNESS: [u8; 32] = [
 	217, 153, 138, 37, 48, 192, 248, 0,
 ];
 
-fn make_pre_digest(
-	authority_index: sp_consensus_babe::AuthorityIndex,
-	slot_number: sp_consensus_babe::SlotNumber,
-	vrf_output: RawVRFOutput,
-	vrf_proof: RawVRFProof,
-) -> Digest {
-	let digest_data = sp_consensus_babe::digests::RawPreDigest::Primary(
-		sp_consensus_babe::digests::RawPrimaryPreDigest {
-			authority_index,
-			slot_number,
-			vrf_output,
-			vrf_proof,
-		}
-	);
-	let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode());
-	Digest { logs: vec![log] }
-}
-
 #[test]
 fn empty_randomness_is_correct() {
 	let s = compute_randomness([0; RANDOMNESS_LENGTH], 0, std::iter::empty(), None);
@@ -132,3 +113,24 @@ fn authority_index() {
 			"Trivially invalid authorities are ignored")
 	})
 }
+
+#[test]
+fn can_predict_next_epoch_change() {
+	new_test_ext(vec![]).execute_with(|| {
+		assert_eq!(<Test as Trait>::EpochDuration::get(), 3);
+		// this sets the genesis slot to 6;
+		go_to_block(1, 6);
+		assert_eq!(Babe::genesis_slot(), 6);
+		assert_eq!(Babe::current_slot(), 6);
+		assert_eq!(Babe::epoch_index(), 0);
+
+		progress_to_block(5);
+
+		assert_eq!(Babe::epoch_index(), 5 / 3);
+		assert_eq!(Babe::current_slot(), 10);
+
+		// next epoch change will be at
+		assert_eq!(Babe::current_epoch_start(), 9); // next change will be 12, 2 slots from now
+		assert_eq!(Babe::next_expected_epoch_change(System::block_number()), Some(5 + 2));
+	})
+}

substrate/frame/elections-phragmen/src/lib.rs

@@ -94,7 +94,7 @@ use frame_support::{
 		ChangeMembers, OnUnbalanced, WithdrawReason, Contains, BalanceStatus
 	}
 };
-use sp_phragmen::ExtendedBalance;
+use sp_phragmen::{build_support_map, ExtendedBalance};
 use frame_system::{self as system, ensure_signed, ensure_root};
 
 const MODULE_ID: LockIdentifier = *b"phrelect";
@@ -692,12 +692,18 @@ impl<T: Trait> Module<T> {
 				.filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } )
 				.collect::<Vec<T::AccountId>>();
 
-			let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>(
-				&new_set,
-				&phragmen_result.assignments,
-				Self::locked_stake_of,
+			let stake_of = |who: &T::AccountId| -> ExtendedBalance {
+				<T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(
+					Self::locked_stake_of(who)
+				) as ExtendedBalance
+			};
+			let staked_assignments = sp_phragmen::assignment_ratio_to_staked(
+				phragmen_result.assignments,
+				stake_of,
 			);
 
+			let (support_map, _) = build_support_map::<T::AccountId>(&new_set, &staked_assignments);
+
 			let to_balance = |e: ExtendedBalance|
 				<T::CurrencyToVote as Convert<ExtendedBalance, BalanceOf<T>>>::convert(e);
 			let new_set_with_stake = new_set

substrate/frame/im-online/src/mock.rs

@@ -141,6 +141,7 @@ impl pallet_session::Trait for Runtime {
 	type Keys = UintAuthorityId;
 	type Event = ();
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+	type NextSessionRotation = pallet_session::PeriodicSessions<Period, Offset>;
 }
 
 impl pallet_session::historical::Trait for Runtime {

substrate/frame/offences/src/lib.rs

@@ -26,11 +26,12 @@ mod tests;
 
 use sp_std::vec::Vec;
 use frame_support::{
-	decl_module, decl_event, decl_storage, Parameter,
+	decl_module, decl_event, decl_storage, Parameter, debug,
 	weights::{Weight, SimpleDispatchInfo, WeighData},
 };
-use sp_runtime::traits::Hash;
+use sp_runtime::{traits::Hash, Perbill};
 use sp_staking::{
+	SessionIndex,
 	offence::{Offence, ReportOffence, Kind, OnOffenceHandler, OffenceDetails, OffenceError},
 };
 use codec::{Encode, Decode};
@@ -42,6 +43,13 @@ type OpaqueTimeSlot = Vec<u8>;
 /// A type alias for a report identifier.
 type ReportIdOf<T> = <T as frame_system::Trait>::Hash;
 
+/// Type of data stored as a deferred offence
+type DeferredOffenceOf<T> = (
+	Vec<OffenceDetails<<T as frame_system::Trait>::AccountId, <T as Trait>::IdentificationTuple>>,
+	Vec<Perbill>,
+	SessionIndex,
+);
+
 /// Offences trait
 pub trait Trait: frame_system::Trait {
 	/// The overarching event type.
@@ -59,6 +67,10 @@ decl_storage! {
 			map hasher(twox_64_concat) ReportIdOf<T>
 			=> Option<OffenceDetails<T::AccountId, T::IdentificationTuple>>;
 
+		/// Deferred reports that have been rejected by the offence handler and need to be submitted
+		/// at a later time.
+		DeferredOffences get(deferred_offences): Vec<DeferredOffenceOf<T>>;
+
 		/// A vector of reports of the same kind that happened at the same time slot.
 		ConcurrentReportsIndex:
 			double_map hasher(twox_64_concat) Kind, hasher(twox_64_concat) OpaqueTimeSlot
@@ -77,13 +89,13 @@ decl_storage! {
 decl_event!(
 	pub enum Event {
 		/// There is an offence reported of the given `kind` happened at the `session_index` and
-		/// (kind-specific) time slot. This event is not deposited for duplicate slashes.
-		Offence(Kind, OpaqueTimeSlot),
+		/// (kind-specific) time slot. This event is not deposited for duplicate slashes. last
+		/// element indicates of the offence was applied (true) or queued (false).
+		Offence(Kind, OpaqueTimeSlot, bool),
 	}
 );
 
 decl_module! {
-	/// Offences module, currently just responsible for taking offence reports.
 	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
 		fn deposit_event() = default;
 
@@ -94,6 +106,27 @@ decl_module! {
 
 			SimpleDispatchInfo::default().weigh_data(())
 		}
+
+		fn on_initialize(now: T::BlockNumber) -> Weight {
+			// only decode storage if we can actually submit anything again.
+			if T::OnOffenceHandler::can_report() {
+				<DeferredOffences<T>>::mutate(|deferred| {
+					// keep those that fail to be reported again. An error log is emitted here; this
+					// should not happen if staking's `can_report` is implemented properly.
+					deferred.retain(|(o, p, s)| {
+						T::OnOffenceHandler::on_offence(&o, &p, *s).map_err(|_| {
+							debug::native::error!(
+								target: "pallet-offences",
+								"re-submitting a deferred slash returned Err at {}. This should not happen with pallet-staking",
+								now,
+							);
+						}).is_err()
+					})
+				})
+			}
+
+			SimpleDispatchInfo::default().weigh_data(())
+		}
 	}
 }
 
@@ -119,9 +152,6 @@ where
 			None => return Err(OffenceError::DuplicateReport),
 		};
 
-		// Deposit the event.
-		Self::deposit_event(Event::Offence(O::ID, time_slot.encode()));
-
 		let offenders_count = concurrent_offenders.len() as u32;
 
 		// The amount new offenders are slashed
@@ -130,17 +160,42 @@ where
 		let slash_perbill: Vec<_> = (0..concurrent_offenders.len())
 			.map(|_| new_fraction.clone()).collect();
 
-		T::OnOffenceHandler::on_offence(
+		let applied = Self::report_or_store_offence(
 			&concurrent_offenders,
 			&slash_perbill,
 			offence.session_index(),
 		);
 
+		// Deposit the event.
+		Self::deposit_event(Event::Offence(O::ID, time_slot.encode(), applied));
+
 		Ok(())
 	}
 }
 
 impl<T: Trait> Module<T> {
+	/// Tries (without checking) to report an offence. Stores them in [`DeferredOffences`] in case
+	/// it fails. Returns false in case it has to store the offence.
+	fn report_or_store_offence(
+		concurrent_offenders: &[OffenceDetails<T::AccountId, T::IdentificationTuple>],
+		slash_perbill: &[Perbill],
+		session_index: SessionIndex,
+	) -> bool {
+		match T::OnOffenceHandler::on_offence(
+			&concurrent_offenders,
+			&slash_perbill,
+			session_index,
+		) {
+			Ok(_) => true,
+			Err(_) => {
+				<DeferredOffences<T>>::mutate(|d|
+					d.push((concurrent_offenders.to_vec(), slash_perbill.to_vec(), session_index))
+				);
+				false
+			}
+		}
+	}
+
 	/// Compute the ID for the given report properties.
 	///
 	/// The report id depends on the offence kind, time slot and the id of offender.

substrate/frame/offences/src/mock.rs

@@ -43,6 +43,7 @@ pub struct OnOffenceHandler;
 
 thread_local! {
 	pub static ON_OFFENCE_PERBILL: RefCell<Vec<Perbill>> = RefCell::new(Default::default());
+	pub static CAN_REPORT: RefCell<bool> = RefCell::new(true);
 }
 
 impl<Reporter, Offender> offence::OnOffenceHandler<Reporter, Offender> for OnOffenceHandler {
@@ -50,11 +51,25 @@ impl<Reporter, Offender> offence::OnOffenceHandler<Reporter, Offender> for OnOff
 		_offenders: &[OffenceDetails<Reporter, Offender>],
 		slash_fraction: &[Perbill],
 		_offence_session: SessionIndex,
-	) {
-		ON_OFFENCE_PERBILL.with(|f| {
-			*f.borrow_mut() = slash_fraction.to_vec();
-		});
+	) -> Result<(), ()> {
+		if <Self as offence::OnOffenceHandler<Reporter, Offender>>::can_report() {
+			ON_OFFENCE_PERBILL.with(|f| {
+				*f.borrow_mut() = slash_fraction.to_vec();
+			});
+
+			Ok(())
+		} else {
+			Err(())
+		}
 	}
+
+	fn can_report() -> bool {
+		CAN_REPORT.with(|c| *c.borrow())
+	}
+}
+
+pub fn set_can_report(can_report: bool) {
+	CAN_REPORT.with(|c| *c.borrow_mut() = can_report);
 }
 
 pub fn with_on_offence_fractions<R, F: FnOnce(&mut Vec<Perbill>) -> R>(f: F) -> R {

substrate/frame/offences/src/tests.rs

@@ -21,9 +21,10 @@
 use super::*;
 use crate::mock::{
 	Offences, System, Offence, TestEvent, KIND, new_test_ext, with_on_offence_fractions,
-	offence_reports,
+	offence_reports, set_can_report,
 };
 use sp_runtime::Perbill;
+use frame_support::traits::OnInitialize;
 use frame_system::{EventRecord, Phase};
 
 #[test]
@@ -130,7 +131,7 @@ fn should_deposit_event() {
 			System::events(),
 			vec![EventRecord {
 				phase: Phase::Initialization,
-				event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode())),
+				event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode(), true)),
 				topics: vec![],
 			}]
 		);
@@ -165,7 +166,7 @@ fn doesnt_deposit_event_for_dups() {
 			System::events(),
 			vec![EventRecord {
 				phase: Phase::Initialization,
-				event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode())),
+				event: TestEvent::offences(crate::Event::Offence(KIND, time_slot.encode(), true)),
 				topics: vec![],
 			}]
 		);
@@ -212,3 +213,54 @@ fn should_properly_count_offences() {
 		);
 	});
 }
+
+#[test]
+fn should_queue_and_resubmit_rejected_offence() {
+	new_test_ext().execute_with(|| {
+		set_can_report(false);
+
+		// will get deferred
+		let offence = Offence {
+			validator_set_count: 5,
+			time_slot: 42,
+			offenders: vec![5],
+		};
+		Offences::report_offence(vec![], offence).unwrap();
+		assert_eq!(Offences::deferred_offences().len(), 1);
+		// event also indicates unapplied.
+		assert_eq!(
+			System::events(),
+			vec![EventRecord {
+				phase: Phase::Initialization,
+				event: TestEvent::offences(crate::Event::Offence(KIND, 42u128.encode(), false)),
+				topics: vec![],
+			}]
+		);
+
+		// will not dequeue
+		Offences::on_initialize(2);
+
+		// again
+		let offence = Offence {
+			validator_set_count: 5,
+			time_slot: 62,
+			offenders: vec![5],
+		};
+		Offences::report_offence(vec![], offence).unwrap();
+		assert_eq!(Offences::deferred_offences().len(), 2);
+
+		set_can_report(true);
+
+		// can be submitted
+		let offence = Offence {
+			validator_set_count: 5,
+			time_slot: 72,
+			offenders: vec![5],
+		};
+		Offences::report_offence(vec![], offence).unwrap();
+		assert_eq!(Offences::deferred_offences().len(), 2);
+
+		Offences::on_initialize(3);
+		assert_eq!(Offences::deferred_offences().len(), 0);
+	})
+}

substrate/frame/session/src/lib.rs

@@ -102,13 +102,15 @@
 use sp_std::{prelude::*, marker::PhantomData, ops::{Sub, Rem}};
 use codec::Decode;
 use sp_runtime::{KeyTypeId, Perbill, RuntimeAppPublic, BoundToRuntimeAppPublic};
-use sp_runtime::traits::{Convert, Zero, Member, OpaqueKeys};
+use sp_runtime::traits::{Convert, Zero, Member, OpaqueKeys, Saturating};
 use sp_staking::SessionIndex;
 use frame_support::{
 	ensure, decl_module, decl_event, decl_storage, decl_error, ConsensusEngineId, Parameter,
-	weights::{Weight, SimpleDispatchInfo, WeighData},
-	traits::{Get, FindAuthor, ValidatorRegistration},
+	traits::{
+		Get, FindAuthor, ValidatorRegistration, EstimateNextSessionRotation, EstimateNextNewSession,
+	},
 	dispatch::{self, DispatchResult, DispatchError},
+	weights::{Weight, SimpleDispatchInfo, WeighData},
 };
 use frame_system::{self as system, ensure_signed};
 
@@ -147,6 +149,29 @@ impl<
 	}
 }
 
+impl<
+	BlockNumber: Rem<Output=BlockNumber> + Sub<Output=BlockNumber> + Zero + PartialOrd + Saturating + Clone,
+	Period: Get<BlockNumber>,
+	Offset: Get<BlockNumber>,
+> EstimateNextSessionRotation<BlockNumber> for PeriodicSessions<Period, Offset> {
+	fn estimate_next_session_rotation(now: BlockNumber) -> Option<BlockNumber> {
+		let offset = Offset::get();
+		let period = Period::get();
+		Some(if now > offset {
+			let block_after_last_session = (now.clone() - offset) % period.clone();
+			if block_after_last_session > Zero::zero() {
+				now.saturating_add(
+					period.saturating_sub(block_after_last_session)
+				)
+			} else {
+				Zero::zero()
+			}
+		} else {
+			offset
+		})
+	}
+}
+
 /// A trait for managing creation of new validator set.
 pub trait SessionManager<ValidatorId> {
 	/// Plan a new session, and optionally provide the new validator set.
@@ -330,6 +355,11 @@ pub trait Trait: frame_system::Trait {
 	/// Indicator for when to end the session.
 	type ShouldEndSession: ShouldEndSession<Self::BlockNumber>;
 
+	/// Something that can predict the next session rotation. This should typically come from the
+	/// same logical unit that provides [`ShouldEndSession`], yet, it gives a best effort estimate.
+	/// It is helpful to implement [`EstimateNextNewSession`].
+	type NextSessionRotation: EstimateNextSessionRotation<Self::BlockNumber>;
+
 	/// Handler for managing new session.
 	type SessionManager: SessionManager<Self::ValidatorId>;
 
@@ -735,3 +765,11 @@ impl<T: Trait, Inner: FindAuthor<u32>> FindAuthor<T::ValidatorId>
 		validators.get(i as usize).map(|k| k.clone())
 	}
 }
+
+impl<T: Trait> EstimateNextNewSession<T::BlockNumber> for Module<T> {
+	/// This session module always calls new_session and next_session at the same time, hence we
+	/// do a simple proxy and pass the function to next rotation.
+	fn estimate_next_new_session(now: T::BlockNumber) -> Option<T::BlockNumber> {
+		T::NextSessionRotation::estimate_next_session_rotation(now)
+	}
+}

substrate/frame/session/src/mock.rs

@@ -205,6 +205,7 @@ impl Trait for Test {
 	type Keys = MockSessionKeys;
 	type Event = ();
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+	type NextSessionRotation = ();
 }
 
 #[cfg(feature = "historical")]

substrate/frame/staking/Cargo.toml

@@ -11,7 +11,6 @@ description = "FRAME pallet staking"
 [dependencies]
 serde = { version = "1.0.101", optional = true }
 codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] }
-sp-keyring = { version = "2.0.0-alpha.5", optional = true, path = "../../primitives/keyring" }
 sp-std = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/std" }
 sp-phragmen = { version = "2.0.0-alpha.5", default-features = false, path = "../../primitives/phragmen" }
 sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-alpha.5"}
@@ -21,7 +20,15 @@ frame-support = { version = "2.0.0-alpha.5", default-features = false, path = ".
 frame-system = { version = "2.0.0-alpha.5", default-features = false, path = "../system" }
 pallet-session = { version = "2.0.0-alpha.5", features = ["historical"], path = "../session", default-features = false }
 pallet-authorship = { version = "2.0.0-alpha.5", default-features = false, path = "../authorship" }
+sp-application-crypto = { version = "2.0.0-alpha.4", default-features = false, path = "../../primitives/application-crypto" }
+static_assertions = "1.1.0"
 
+# Optional imports for tesing-utils feature
+pallet-indices = { version = "2.0.0-alpha.4", optional = true, path = "../indices", default-features = false }
+sp-core = { version = "2.0.0-alpha.4", optional = true, path = "../../primitives/core", default-features = false }
+rand = { version = "0.7.3", optional = true, default-features = false }
+
+# Optional imports for benchmarking
 frame-benchmarking = { version = "2.0.0-alpha.5", default-features = false, path = "../benchmarking", optional = true }
 rand_chacha = { version = "0.2", default-features = false, optional = true }
 
@@ -33,13 +40,20 @@ pallet-staking-reward-curve = { version = "2.0.0-alpha.5",  path = "../staking/r
 substrate-test-utils = { version = "2.0.0-alpha.5", path = "../../test-utils" }
 frame-benchmarking = { version = "2.0.0-alpha.5", path = "../benchmarking" }
 rand_chacha = { version = "0.2" }
+parking_lot = "0.10.0"
+env_logger = "0.7.1"
 hex = "0.4"
 
 [features]
+testing-utils = [
+	"std",
+	"pallet-indices/std",
+	"sp-core/std",
+	"rand/std",
+]
 default = ["std"]
 std = [
 	"serde",
-	"sp-keyring",
 	"codec/std",
 	"sp-std/std",
 	"sp-phragmen/std",
@@ -50,6 +64,8 @@ std = [
 	"pallet-session/std",
 	"frame-system/std",
 	"pallet-authorship/std",
+	"sp-application-crypto/std",
+	"sp-core/std",
 ]
 runtime-benchmarks = [
 	"rand_chacha",

substrate/frame/staking/fuzz/.gitignore

@@ -0,0 +1,4 @@
+
+target
+corpus
+artifacts

substrate/frame/staking/fuzz/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "pallet-staking-fuzz"
+version = "0.0.0"
+authors = ["Automatically generated"]
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.3"
+codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive"] }
+pallet-staking = { version = "2.0.0-alpha.2", path = "..", features = ["testing-utils"] }
+pallet-staking-reward-curve = { version = "2.0.0-alpha.2",  path = "../reward-curve" }
+pallet-session = { version = "2.0.0-alpha.2", path = "../../session" }
+pallet-indices = { version = "2.0.0-alpha.2", path = "../../indices" }
+pallet-balances = { version = "2.0.0-alpha.2", path = "../../balances" }
+pallet-timestamp = { version = "2.0.0-alpha.2", path = "../../timestamp" }
+frame-system = { version = "2.0.0-alpha.2", path = "../../system" }
+frame-support = { version = "2.0.0-alpha.2", path = "../../support" }
+sp-std = { version = "2.0.0-alpha.2", path = "../../../primitives/std" }
+sp-io ={ version = "2.0.0-alpha.2", path = "../../../primitives/io" }
+sp-core = { version = "2.0.0-alpha.2", path = "../../../primitives/core" }
+sp-phragmen = { version = "2.0.0-alpha.2", path = "../../../primitives/phragmen" }
+sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" }
+rand = "0.7.3"
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "submit_solution"
+path = "fuzz_targets/submit_solution.rs"

substrate/frame/staking/fuzz/fuzz_targets/mock.rs

@@ -0,0 +1,182 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Mock file for staking fuzzing.
+
+use sp_runtime::traits::{Convert, SaturatedConversion};
+use frame_support::{impl_outer_origin, impl_outer_dispatch, parameter_types};
+
+type AccountId = u64;
+type AccountIndex = u32;
+type BlockNumber = u64;
+type Balance = u64;
+
+type System = frame_system::Module<Test>;
+type Balances = pallet_balances::Module<Test>;
+type Staking = pallet_staking::Module<Test>;
+type Indices = pallet_indices::Module<Test>;
+type Session = pallet_session::Module<Test>;
+
+impl_outer_origin! {
+	pub enum Origin for Test  where system = frame_system {}
+}
+
+impl_outer_dispatch! {
+	pub enum Call for Test where origin: Origin {
+		staking::Staking,
+	}
+}
+
+pub struct CurrencyToVoteHandler;
+impl Convert<u64, u64> for CurrencyToVoteHandler {
+	fn convert(x: u64) -> u64 {
+		x
+	}
+}
+impl Convert<u128, u64> for CurrencyToVoteHandler {
+	fn convert(x: u128) -> u64 {
+		x.saturated_into()
+	}
+}
+
+#[derive(Clone, Eq, PartialEq, Debug)]
+pub struct Test;
+
+impl frame_system::Trait for Test {
+	type Origin = Origin;
+	type Index = AccountIndex;
+	type BlockNumber = BlockNumber;
+	type Call = Call;
+	type Hash = sp_core::H256;
+	type Hashing = ::sp_runtime::traits::BlakeTwo256;
+	type AccountId = AccountId;
+	type Lookup = Indices;
+	type Header = sp_runtime::testing::Header;
+	type Event = ();
+	type BlockHashCount = ();
+	type MaximumBlockWeight = ();
+	type AvailableBlockRatio = ();
+	type MaximumBlockLength = ();
+	type Version = ();
+	type ModuleToIndex = ();
+	type AccountData = pallet_balances::AccountData<u64>;
+	type OnNewAccount = ();
+	type OnKilledAccount = (Balances,);
+}
+parameter_types! {
+	pub const ExistentialDeposit: Balance = 10;
+}
+impl pallet_balances::Trait for Test {
+	type Balance = Balance;
+	type Event = ();
+	type DustRemoval = ();
+	type ExistentialDeposit = ExistentialDeposit;
+	type AccountStore = System;
+}
+impl pallet_indices::Trait for Test {
+	type AccountIndex = AccountIndex;
+	type Event = ();
+	type Currency = Balances;
+	type Deposit = ();
+}
+parameter_types! {
+	pub const MinimumPeriod: u64 = 5;
+}
+impl pallet_timestamp::Trait for Test {
+	type Moment = u64;
+	type OnTimestampSet = ();
+	type MinimumPeriod = MinimumPeriod;
+}
+impl pallet_session::historical::Trait for Test {
+	type FullIdentification = pallet_staking::Exposure<AccountId, Balance>;
+	type FullIdentificationOf = pallet_staking::ExposureOf<Test>;
+}
+
+sp_runtime::impl_opaque_keys! {
+	pub struct SessionKeys {
+		pub foo: sp_runtime::testing::UintAuthorityId,
+	}
+}
+
+pub struct TestSessionHandler;
+impl pallet_session::SessionHandler<AccountId> for TestSessionHandler {
+	const KEY_TYPE_IDS: &'static [sp_runtime::KeyTypeId] = &[];
+
+	fn on_genesis_session<Ks: sp_runtime::traits::OpaqueKeys>(_validators: &[(AccountId, Ks)]) {}
+
+	fn on_new_session<Ks: sp_runtime::traits::OpaqueKeys>(
+		_: bool,
+		_: &[(AccountId, Ks)],
+		_: &[(AccountId, Ks)],
+	) {}
+
+	fn on_disabled(_: usize) {}
+}
+
+impl pallet_session::Trait for Test {
+	type SessionManager = pallet_session::historical::NoteHistoricalRoot<Test, Staking>;
+	type Keys = SessionKeys;
+	type ShouldEndSession = pallet_session::PeriodicSessions<(), ()>;
+	type NextSessionRotation = pallet_session::PeriodicSessions<(), ()>;
+	type SessionHandler = TestSessionHandler;
+	type Event = ();
+	type ValidatorId = AccountId;
+	type ValidatorIdOf = pallet_staking::StashOf<Test>;
+	type DisabledValidatorsThreshold = ();
+}
+pallet_staking_reward_curve::build! {
+	const I_NPOS: sp_runtime::curve::PiecewiseLinear<'static> = curve!(
+		min_inflation: 0_025_000,
+		max_inflation: 0_100_000,
+		ideal_stake: 0_500_000,
+		falloff: 0_050_000,
+		max_piece_count: 40,
+		test_precision: 0_005_000,
+	);
+}
+parameter_types! {
+	pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS;
+	pub const MaxNominatorRewardedPerValidator: u32 = 64;
+}
+
+pub type Extrinsic = sp_runtime::testing::TestXt<Call, ()>;
+type SubmitTransaction = frame_system::offchain::TransactionSubmitter<
+	sp_runtime::testing::UintAuthorityId,
+	Test,
+	Extrinsic,
+>;
+
+impl pallet_staking::Trait for Test {
+	type Currency = Balances;
+	type Time = pallet_timestamp::Module<Self>;
+	type CurrencyToVote = CurrencyToVoteHandler;
+	type RewardRemainder = ();
+	type Event = ();
+	type Slash = ();
+	type Reward = ();
+	type SessionsPerEra = ();
+	type SlashDeferDuration = ();
+	type SlashCancelOrigin = frame_system::EnsureRoot<Self::AccountId>;
+	type BondingDuration = ();
+	type SessionInterface = Self;
+	type RewardCurve = RewardCurve;
+	type NextNewSession = Session;
+	type ElectionLookahead = ();
+	type Call = Call;
+	type SubmitTransaction = SubmitTransaction;
+	type KeyType = sp_runtime::testing::UintAuthorityId;
+	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
+}

substrate/frame/staking/fuzz/fuzz_targets/submit_solution.rs

@@ -0,0 +1,130 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Fuzzing for staking pallet.
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use mock::Test;
+use pallet_staking::testing_utils::{
+	self, USER, get_seq_phragmen_solution, get_weak_solution, setup_chain_stakers,
+	set_validator_count, signed_account,
+};
+use frame_support::assert_ok;
+use sp_runtime::{traits::Dispatchable, DispatchError};
+
+mod mock;
+
+#[repr(u32)]
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy)]
+enum Mode {
+	/// Initial submission. This will be rather cheap.
+	InitialSubmission,
+	/// A better submission that will replace the previous ones. This is the most expensive.
+	StrongerSubmission,
+	/// A weak submission that will be rejected. This will be rather cheap.
+	WeakerSubmission,
+}
+
+pub fn new_test_ext() -> Result<sp_io::TestExternalities, std::string::String> {
+	frame_system::GenesisConfig::default().build_storage::<mock::Test>().map(Into::into)
+}
+
+fuzz_target!(|do_reduce: bool| {
+    let ext = new_test_ext();
+	let mode: Mode = unsafe { std::mem::transmute(testing_utils::random(0, 2)) };
+	let num_validators = testing_utils::random(50, 500);
+	let num_nominators = testing_utils::random(200, 2000);
+	let edge_per_voter = testing_utils::random(1, 16);
+	let to_elect = testing_utils::random(10, num_validators);
+
+	println!("+++ instance with params {} / {} / {} / {:?} / {}",
+		num_nominators,
+		num_validators,
+		edge_per_voter,
+		mode,
+		to_elect,
+	);
+
+	ext.unwrap_or_default().execute_with(|| {
+		// initial setup
+		set_validator_count::<Test>(to_elect);
+		setup_chain_stakers::<Test>(
+			num_validators,
+			num_nominators,
+			edge_per_voter,
+		);
+
+		println!("++ Chain setup done.");
+
+		// stuff to submit
+		let (winners, compact, score) = match mode {
+			Mode::InitialSubmission => {
+				/* No need to setup anything */
+				get_seq_phragmen_solution::<Test>(do_reduce)
+			},
+			Mode::StrongerSubmission => {
+				let (winners, compact, score) = get_weak_solution::<Test>(false);
+				assert_ok!(
+					<pallet_staking::Module<Test>>::submit_election_solution(
+						signed_account::<Test>(USER),
+						winners,
+						compact,
+						score,
+					)
+				);
+				get_seq_phragmen_solution::<Test>(do_reduce)
+			},
+			Mode::WeakerSubmission => {
+				let (winners, compact, score) = get_seq_phragmen_solution::<Test>(do_reduce);
+				assert_ok!(
+					<pallet_staking::Module<Test>>::submit_election_solution(
+						signed_account::<Test>(USER),
+						winners,
+						compact,
+						score,
+					)
+				);
+				get_weak_solution::<Test>(false)
+			}
+		};
+
+		println!("++ Submission ready.");
+
+		// must have chosen correct number of winners.
+		assert_eq!(winners.len() as u32, <pallet_staking::Module<Test>>::validator_count());
+
+		// final call and origin
+		let call = pallet_staking::Call::<Test>::submit_election_solution(
+			winners,
+			compact,
+			score,
+		);
+		let caller = signed_account::<Test>(USER);
+
+		// actually submit
+		match mode {
+			Mode::WeakerSubmission => {
+				assert_eq!(
+					call.dispatch(caller.into()).unwrap_err(),
+					DispatchError::Module { index: 0, error: 11, message: Some("PhragmenWeakSubmission") },
+				);
+			},
+			_ => assert_ok!(call.dispatch(caller.into())),
+		};
+	})
+});

substrate/frame/staking/src/benchmarking.rs

@@ -402,7 +402,7 @@ mod tests {
 
 	#[test]
 	fn create_validators_with_nominators_for_era_works() {
-		ExtBuilder::default().stakers(false).build().execute_with(|| {
+		ExtBuilder::default().has_stakers(false).build().execute_with(|| {
 			let v = 10;
 			let n = 100;
 
@@ -418,7 +418,7 @@ mod tests {
 
 	#[test]
 	fn create_validator_with_nominators_works() {
-		ExtBuilder::default().stakers(false).build().execute_with(|| {
+		ExtBuilder::default().has_stakers(false).build().execute_with(|| {
 			let n = 10;
 
 			let validator = create_validator_with_nominators::<Test>(
@@ -441,7 +441,7 @@ mod tests {
 
 	#[test]
 	fn create_nominator_with_validators_works() {
-		ExtBuilder::default().stakers(false).build().execute_with(|| {
+		ExtBuilder::default().has_stakers(false).build().execute_with(|| {
 			let v = 5;
 
 			let (nominator, validators) = create_nominator_with_validators::<Test>(v).unwrap();

substrate/frame/staking/src/mock.rs

@@ -17,57 +17,74 @@
 //! Test utilities
 
 use std::{collections::{HashSet, HashMap}, cell::RefCell};
-use sp_runtime::{Perbill, KeyTypeId};
+use sp_runtime::Perbill;
 use sp_runtime::curve::PiecewiseLinear;
-use sp_runtime::traits::{IdentityLookup, Convert, OpaqueKeys, SaturatedConversion};
-use sp_runtime::testing::{Header, UintAuthorityId};
+use sp_runtime::traits::{IdentityLookup, Convert, SaturatedConversion, Zero};
+use sp_runtime::testing::{Header, UintAuthorityId, TestXt};
 use sp_staking::{SessionIndex, offence::{OffenceDetails, OnOffenceHandler}};
-use sp_core::{H256, crypto::key_types};
-use sp_io;
+use sp_core::H256;
 use frame_support::{
-	assert_ok, impl_outer_origin, parameter_types, StorageValue, StorageMap,
-	StorageDoubleMap, IterableStorageMap,
-	traits::{Currency, Get, FindAuthor, OnFinalize, OnInitialize}, weights::Weight,
+	assert_ok, impl_outer_origin, parameter_types, impl_outer_dispatch, impl_outer_event,
+	StorageValue, StorageMap, StorageDoubleMap, IterableStorageMap,
+	traits::{Currency, Get, FindAuthor, OnFinalize, OnInitialize},
+	weights::Weight,
+};
+use frame_system::offchain::TransactionSubmitter;
+use sp_io;
+use sp_phragmen::{
+	build_support_map, evaluate_support, reduce, ExtendedBalance, StakedAssignment, PhragmenScore,
 };
 use crate::{
 	EraIndex, GenesisConfig, Module, Trait, StakerStatus, ValidatorPrefs, RewardDestination,
-	Nominators, inflation, SessionInterface, Exposure, ErasStakers, ErasRewardPoints
+	Nominators, inflation, SessionInterface, Exposure, ErasStakers, ErasRewardPoints,
+	CompactAssignments, ValidatorIndex, NominatorIndex, Validators, OffchainAccuracy,
 };
 
 /// The AccountId alias in this test module.
-pub type AccountId = u64;
-pub type BlockNumber = u64;
-pub type Balance = u64;
+pub(crate) type AccountId = u64;
+pub(crate) type AccountIndex = u64;
+pub(crate) type BlockNumber = u64;
+pub(crate) type Balance = u64;
 
 /// Simple structure that exposes how u64 currency can be represented as... u64.
 pub struct CurrencyToVoteHandler;
 impl Convert<u64, u64> for CurrencyToVoteHandler {
-	fn convert(x: u64) -> u64 { x }
+	fn convert(x: u64) -> u64 {
+		x
+	}
 }
 impl Convert<u128, u64> for CurrencyToVoteHandler {
-	fn convert(x: u128) -> u64 { x.saturated_into() }
+	fn convert(x: u128) -> u64 {
+		x.saturated_into()
+	}
 }
 
 thread_local! {
 	static SESSION: RefCell<(Vec<AccountId>, HashSet<AccountId>)> = RefCell::new(Default::default());
+	static SESSION_PER_ERA: RefCell<SessionIndex> = RefCell::new(3);
 	static EXISTENTIAL_DEPOSIT: RefCell<u64> = RefCell::new(0);
 	static SLASH_DEFER_DURATION: RefCell<EraIndex> = RefCell::new(0);
+	static ELECTION_LOOKAHEAD: RefCell<BlockNumber> = RefCell::new(0);
+	static PERIOD: RefCell<BlockNumber> = RefCell::new(1);
 }
 
-pub struct TestSessionHandler;
-impl pallet_session::SessionHandler<AccountId> for TestSessionHandler {
-	const KEY_TYPE_IDS: &'static [KeyTypeId] = &[key_types::DUMMY];
+/// Another session handler struct to test on_disabled.
+pub struct OtherSessionHandler;
+impl pallet_session::OneSessionHandler<AccountId> for OtherSessionHandler {
+	type Key = UintAuthorityId;
 
-	fn on_genesis_session<Ks: OpaqueKeys>(_validators: &[(AccountId, Ks)]) {}
+	fn on_genesis_session<'a, I: 'a>(_: I)
+		where I: Iterator<Item=(&'a AccountId, Self::Key)>, AccountId: 'a {}
 
-	fn on_new_session<Ks: OpaqueKeys>(
-		_changed: bool,
-		validators: &[(AccountId, Ks)],
-		_queued_validators: &[(AccountId, Ks)],
-	) {
-		SESSION.with(|x|
-			*x.borrow_mut() = (validators.iter().map(|x| x.0.clone()).collect(), HashSet::new())
-		);
+	fn on_new_session<'a, I: 'a>(_: bool, validators: I, _: I,)
+		where I: Iterator<Item=(&'a AccountId, Self::Key)>, AccountId: 'a
+	{
+		SESSION.with(|x| {
+			*x.borrow_mut() = (
+				validators.map(|x| x.0.clone()).collect(),
+				HashSet::new(),
+			)
+		});
 	}
 
 	fn on_disabled(validator_index: usize) {
@@ -79,6 +96,10 @@ impl pallet_session::SessionHandler<AccountId> for TestSessionHandler {
 	}
 }
 
+impl sp_runtime::BoundToRuntimeAppPublic for OtherSessionHandler {
+	type Public = UintAuthorityId;
+}
+
 pub fn is_disabled(controller: AccountId) -> bool {
 	let stash = Staking::ledger(&controller).unwrap().stash;
 	SESSION.with(|d| d.borrow().1.contains(&stash))
@@ -91,6 +112,32 @@ impl Get<u64> for ExistentialDeposit {
 	}
 }
 
+pub struct SessionsPerEra;
+impl Get<SessionIndex> for SessionsPerEra {
+	fn get() -> SessionIndex {
+		SESSION_PER_ERA.with(|v| *v.borrow())
+	}
+}
+impl Get<BlockNumber> for SessionsPerEra {
+	fn get() -> BlockNumber {
+		SESSION_PER_ERA.with(|v| *v.borrow() as BlockNumber)
+	}
+}
+
+pub struct ElectionLookahead;
+impl Get<BlockNumber> for ElectionLookahead {
+	fn get() -> BlockNumber {
+		ELECTION_LOOKAHEAD.with(|v| *v.borrow())
+	}
+}
+
+pub struct Period;
+impl Get<BlockNumber> for Period {
+	fn get() -> BlockNumber {
+		PERIOD.with(|v| *v.borrow())
+	}
+}
+
 pub struct SlashDeferDuration;
 impl Get<EraIndex> for SlashDeferDuration {
 	fn get() -> EraIndex {
@@ -98,23 +145,47 @@ impl Get<EraIndex> for SlashDeferDuration {
 	}
 }
 
-impl_outer_origin!{
+impl_outer_origin! {
 	pub enum Origin for Test  where system = frame_system {}
 }
 
+impl_outer_dispatch! {
+	pub enum Call for Test where origin: Origin {
+		staking::Staking,
+	}
+}
+
+mod staking {
+	// Re-export needed for `impl_outer_event!`.
+	pub use super::super::*;
+}
+use frame_system as system;
+use pallet_balances as balances;
+use pallet_session as session;
+
+impl_outer_event! {
+	pub enum MetaEvent for Test {
+		system<T>,
+		balances<T>,
+		session,
+		staking<T>,
+	}
+}
+
 /// Author of block is always 11
 pub struct Author11;
 impl FindAuthor<u64> for Author11 {
 	fn find_author<'a, I>(_digests: I) -> Option<u64>
-		where I: 'a + IntoIterator<Item=(frame_support::ConsensusEngineId, &'a [u8])>
+		where I: 'a + IntoIterator<Item = (frame_support::ConsensusEngineId, &'a [u8])>,
 	{
 		Some(11)
 	}
 }
 
 // Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted.
-#[derive(Clone, PartialEq, Eq, Debug)]
+#[derive(Clone, Eq, PartialEq, Debug)]
 pub struct Test;
+
 parameter_types! {
 	pub const BlockHashCount: u64 = 250;
 	pub const MaximumBlockWeight: Weight = 1024;
@@ -123,15 +194,15 @@ parameter_types! {
 }
 impl frame_system::Trait for Test {
 	type Origin = Origin;
-	type Index = u64;
+	type Index = AccountIndex;
 	type BlockNumber = BlockNumber;
-	type Call = ();
+	type Call = Call;
 	type Hash = H256;
 	type Hashing = ::sp_runtime::traits::BlakeTwo256;
 	type AccountId = AccountId;
 	type Lookup = IdentityLookup<Self::AccountId>;
 	type Header = Header;
-	type Event = ();
+	type Event = MetaEvent;
 	type BlockHashCount = BlockHashCount;
 	type MaximumBlockWeight = MaximumBlockWeight;
 	type AvailableBlockRatio = AvailableBlockRatio;
@@ -144,26 +215,31 @@ impl frame_system::Trait for Test {
 }
 impl pallet_balances::Trait for Test {
 	type Balance = Balance;
+	type Event = MetaEvent;
 	type DustRemoval = ();
-	type Event = ();
 	type ExistentialDeposit = ExistentialDeposit;
 	type AccountStore = System;
 }
 parameter_types! {
-	pub const Period: BlockNumber = 1;
 	pub const Offset: BlockNumber = 0;
 	pub const UncleGenerations: u64 = 0;
 	pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(25);
 }
+sp_runtime::impl_opaque_keys! {
+	pub struct SessionKeys {
+		pub other: OtherSessionHandler,
+	}
+}
 impl pallet_session::Trait for Test {
-	type Event = ();
+	type SessionManager = pallet_session::historical::NoteHistoricalRoot<Test, Staking>;
+	type Keys = SessionKeys;
+	type ShouldEndSession = pallet_session::PeriodicSessions<Period, Offset>;
+	type SessionHandler = (OtherSessionHandler,);
+	type Event = MetaEvent;
 	type ValidatorId = AccountId;
 	type ValidatorIdOf = crate::StashOf<Test>;
-	type ShouldEndSession = pallet_session::PeriodicSessions<Period, Offset>;
-	type SessionManager = pallet_session::historical::NoteHistoricalRoot<Test, Staking>;
-	type SessionHandler = TestSessionHandler;
-	type Keys = UintAuthorityId;
 	type DisabledValidatorsThreshold = DisabledValidatorsThreshold;
+	type NextSessionRotation = pallet_session::PeriodicSessions<Period, Offset>;
 }
 
 impl pallet_session::historical::Trait for Test {
@@ -195,17 +271,17 @@ pallet_staking_reward_curve::build! {
 	);
 }
 parameter_types! {
-	pub const SessionsPerEra: SessionIndex = 3;
 	pub const BondingDuration: EraIndex = 3;
 	pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS;
 	pub const MaxNominatorRewardedPerValidator: u32 = 64;
 }
+
 impl Trait for Test {
 	type Currency = Balances;
 	type UnixTime = Timestamp;
 	type CurrencyToVote = CurrencyToVoteHandler;
 	type RewardRemainder = ();
-	type Event = ();
+	type Event = MetaEvent;
 	type Slash = ();
 	type Reward = ();
 	type SessionsPerEra = SessionsPerEra;
@@ -214,11 +290,21 @@ impl Trait for Test {
 	type BondingDuration = BondingDuration;
 	type SessionInterface = Self;
 	type RewardCurve = RewardCurve;
+	type NextNewSession = Session;
+	type ElectionLookahead = ElectionLookahead;
+	type Call = Call;
+	type SubmitTransaction = SubmitTransaction;
 	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
 }
 
+pub type Extrinsic = TestXt<Call, ()>;
+type SubmitTransaction = TransactionSubmitter<(), Test, Extrinsic>;
+
 pub struct ExtBuilder {
-	existential_deposit: u64,
+	session_length: BlockNumber,
+	election_lookahead: BlockNumber,
+	session_per_era: SessionIndex,
+	existential_deposit: Balance,
 	validator_pool: bool,
 	nominate: bool,
 	validator_count: u32,
@@ -226,13 +312,16 @@ pub struct ExtBuilder {
 	slash_defer_duration: EraIndex,
 	fair: bool,
 	num_validators: Option<u32>,
-	invulnerables: Vec<u64>,
-	stakers: bool,
+	invulnerables: Vec<AccountId>,
+	has_stakers: bool,
 }
 
 impl Default for ExtBuilder {
 	fn default() -> Self {
 		Self {
+			session_length: 1,
+			election_lookahead: 0,
+			session_per_era: 3,
 			existential_deposit: 1,
 			validator_pool: false,
 			nominate: true,
@@ -242,7 +331,7 @@ impl Default for ExtBuilder {
 			fair: true,
 			num_validators: None,
 			invulnerables: vec![],
-			stakers: true,
+			has_stakers: true,
 		}
 	}
 }
@@ -284,18 +373,40 @@ impl ExtBuilder {
 		self.invulnerables = invulnerables;
 		self
 	}
-	pub fn set_associated_consts(&self) {
-		EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit);
-		SLASH_DEFER_DURATION.with(|v| *v.borrow_mut() = self.slash_defer_duration);
-	}
-
-	pub fn stakers(mut self, has_stakers: bool) -> Self {
-		self.stakers = has_stakers;
+	pub fn session_per_era(mut self, length: SessionIndex) -> Self {
+		self.session_per_era = length;
 		self
 	}
+	pub fn election_lookahead(mut self, look: BlockNumber) -> Self {
+		self.election_lookahead = look;
+		self
+	}
+	pub fn session_length(mut self, length: BlockNumber) -> Self {
+		self.session_length = length;
+		self
+	}
+	pub fn has_stakers(mut self, has: bool) -> Self {
+		self.has_stakers = has;
+		self
+	}
+	pub fn offchain_phragmen_ext(self) -> Self {
+		self.session_per_era(4)
+			.session_length(5)
+			.election_lookahead(3)
+	}
+	pub fn set_associated_constants(&self) {
+		EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit);
+		SLASH_DEFER_DURATION.with(|v| *v.borrow_mut() = self.slash_defer_duration);
+		SESSION_PER_ERA.with(|v| *v.borrow_mut() = self.session_per_era);
+		ELECTION_LOOKAHEAD.with(|v| *v.borrow_mut() = self.election_lookahead);
+		PERIOD.with(|v| *v.borrow_mut() = self.session_length);
+	}
 	pub fn build(self) -> sp_io::TestExternalities {
-		self.set_associated_consts();
-		let mut storage = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
+		let _ = env_logger::try_init();
+		self.set_associated_constants();
+		let mut storage = frame_system::GenesisConfig::default()
+			.build_storage::<Test>()
+			.unwrap();
 		let balance_factor = if self.existential_deposit > 1 {
 			256
 		} else {
@@ -307,7 +418,7 @@ impl ExtBuilder {
 			.map(|x| ((x + 1) * 10 + 1) as u64)
 			.collect::<Vec<_>>();
 
-		let _ = pallet_balances::GenesisConfig::<Test>{
+		let _ = pallet_balances::GenesisConfig::<Test> {
 			balances: vec![
 				(1, 10 * balance_factor),
 				(2, 20 * balance_factor),
@@ -329,7 +440,7 @@ impl ExtBuilder {
 		}.assimilate_storage(&mut storage);
 
 		let mut stakers = vec![];
-		if self.stakers {
+		if self.has_stakers {
 			let stake_21 = if self.fair { 1000 } else { 2000 };
 			let stake_31 = if self.validator_pool { balance_factor * 1000 } else { 1 };
 			let status_41 = if self.validator_pool {
@@ -355,18 +466,21 @@ impl ExtBuilder {
 			invulnerables: self.invulnerables,
 			slash_reward_fraction: Perbill::from_percent(10),
 			..Default::default()
-		}.assimilate_storage(&mut storage);
+		}
+		.assimilate_storage(&mut storage);
 
 		let _ = pallet_session::GenesisConfig::<Test> {
-			keys: validators.iter().map(|x| (*x, *x, UintAuthorityId(*x))).collect(),
+			keys: validators.iter().map(|x| (
+				*x,
+				*x,
+				SessionKeys { other: UintAuthorityId(*x) }
+			)).collect(),
 		}.assimilate_storage(&mut storage);
 
 		let mut ext = sp_io::TestExternalities::from(storage);
 		ext.execute_with(|| {
 			let validators = Session::validators();
-			SESSION.with(|x|
-				*x.borrow_mut() = (validators.clone(), HashSet::new())
-			);
+			SESSION.with(|x| *x.borrow_mut() = (validators.clone(), HashSet::new()));
 		});
 		ext
 	}
@@ -378,6 +492,10 @@ pub type Session = pallet_session::Module<Test>;
 pub type Timestamp = pallet_timestamp::Module<Test>;
 pub type Staking = Module<Test>;
 
+pub fn active_era() -> EraIndex {
+	Staking::active_era().unwrap().index
+}
+
 pub fn check_exposure_all(era: EraIndex) {
 	ErasStakers::<Test>::iter_prefix(era).for_each(check_exposure)
 }
@@ -390,8 +508,9 @@ pub fn check_nominator_all(era: EraIndex) {
 /// Check for each selected validator: expo.total = Sum(expo.other) + expo.own
 pub fn check_exposure(expo: Exposure<AccountId, Balance>) {
 	assert_eq!(
-		expo.total as u128, expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::<u128>(),
-		"wrong total exposure {:?}", expo,
+		expo.total as u128,
+		expo.own as u128 + expo.others.iter().map(|e| e.value as u128).sum::<u128>(),
+		"wrong total exposure",
 	);
 }
 
@@ -400,17 +519,18 @@ pub fn check_exposure(expo: Exposure<AccountId, Balance>) {
 pub fn check_nominator_exposure(era: EraIndex, stash: AccountId) {
 	assert_is_stash(stash);
 	let mut sum = 0;
-	ErasStakers::<Test>::iter_prefix(era)
-		.for_each(|exposure| {
-			exposure.others.iter()
-				.filter(|i| i.who == stash)
-				.for_each(|i| sum += i.value)
-		});
+	Session::validators()
+		.iter()
+		.map(|v| Staking::eras_stakers(era, v))
+		.for_each(|e| e.others.iter().filter(|i| i.who == stash).for_each(|i| sum += i.value));
 	let nominator_stake = Staking::slashable_balance_of(&stash);
 	// a nominator cannot over-spend.
 	assert!(
 		nominator_stake >= sum,
-		"failed: Nominator({}) stake({}) >= sum divided({})", stash, nominator_stake, sum,
+		"failed: Nominator({}) stake({}) >= sum divided({})",
+		stash,
+		nominator_stake,
+		sum,
 	);
 }
 
@@ -426,20 +546,41 @@ pub fn assert_ledger_consistent(stash: AccountId) {
 	assert_eq!(real_total, ledger.total);
 }
 
-pub fn bond_validator(acc: u64, val: u64) {
-	// a = controller
-	// a + 1 = stash
-	let _ = Balances::make_free_balance_be(&(acc + 1), val);
-	assert_ok!(Staking::bond(Origin::signed(acc + 1), acc, val, RewardDestination::Controller));
-	assert_ok!(Staking::validate(Origin::signed(acc), ValidatorPrefs::default()));
+pub fn bond_validator(stash: u64, ctrl: u64, val: u64) {
+	let _ = Balances::make_free_balance_be(&stash, val);
+	assert_ok!(Staking::bond(
+		Origin::signed(stash),
+		ctrl,
+		val,
+		RewardDestination::Controller,
+	));
+	assert_ok!(Staking::validate(
+		Origin::signed(ctrl),
+		ValidatorPrefs::default()
+	));
 }
 
-pub fn bond_nominator(acc: u64, val: u64, target: Vec<u64>) {
-	// a = controller
-	// a + 1 = stash
-	let _ = Balances::make_free_balance_be(&(acc + 1), val);
-	assert_ok!(Staking::bond(Origin::signed(acc + 1), acc, val, RewardDestination::Controller));
-	assert_ok!(Staking::nominate(Origin::signed(acc), target));
+pub fn bond_nominator(stash: u64, ctrl: u64, val: u64, target: Vec<u64>) {
+	let _ = Balances::make_free_balance_be(&stash, val);
+	assert_ok!(Staking::bond(
+		Origin::signed(stash),
+		ctrl,
+		val,
+		RewardDestination::Controller,
+	));
+	assert_ok!(Staking::nominate(Origin::signed(ctrl), target));
+}
+
+pub fn run_to_block(n: BlockNumber) {
+	Staking::on_finalize(System::block_number());
+	for b in System::block_number() + 1..=n {
+		System::set_block_number(b);
+		Session::on_initialize(b);
+		Staking::on_initialize(b);
+		if b != n {
+			Staking::on_finalize(System::block_number());
+		}
+	}
 }
 
 pub fn advance_session() {
@@ -448,19 +589,21 @@ pub fn advance_session() {
 }
 
 pub fn start_session(session_index: SessionIndex) {
+	assert_eq!(<Period as Get<BlockNumber>>::get(), 1, "start_session can only be used with session length 1.");
 	for i in Session::current_index()..session_index {
 		Staking::on_finalize(System::block_number());
 		System::set_block_number((i + 1).into());
 		Timestamp::set_timestamp(System::block_number() * 1000);
 		Session::on_initialize(System::block_number());
+		Staking::on_initialize(System::block_number());
 	}
 
 	assert_eq!(Session::current_index(), session_index);
 }
 
 pub fn start_era(era_index: EraIndex) {
-	start_session((era_index * 3).into());
-	assert_eq!(Staking::active_era().unwrap().index, era_index);
+	start_session((era_index * <SessionsPerEra as Get<u32>>::get()).into());
+	assert_eq!(Staking::current_era().unwrap(), era_index);
 }
 
 pub fn current_total_payout_for_duration(duration: u64) -> u64 {
@@ -473,33 +616,45 @@ pub fn current_total_payout_for_duration(duration: u64) -> u64 {
 }
 
 pub fn reward_all_elected() {
-	let rewards = <Test as Trait>::SessionInterface::validators().into_iter()
+	let rewards = <Test as Trait>::SessionInterface::validators()
+		.into_iter()
 		.map(|v| (v, 1));
 
 	<Module<Test>>::reward_by_ids(rewards)
 }
 
 pub fn validator_controllers() -> Vec<AccountId> {
-	Session::validators().into_iter().map(|s| Staking::bonded(&s).expect("no controller for validator")).collect()
+	Session::validators()
+		.into_iter()
+		.map(|s| Staking::bonded(&s).expect("no controller for validator"))
+		.collect()
 }
 
 pub fn on_offence_in_era(
-	offenders: &[OffenceDetails<AccountId, pallet_session::historical::IdentificationTuple<Test>>],
+	offenders: &[OffenceDetails<
+		AccountId,
+		pallet_session::historical::IdentificationTuple<Test>,
+	>],
 	slash_fraction: &[Perbill],
 	era: EraIndex,
 ) {
 	let bonded_eras = crate::BondedEras::get();
 	for &(bonded_era, start_session) in bonded_eras.iter() {
 		if bonded_era == era {
-			Staking::on_offence(offenders, slash_fraction, start_session);
-			return
+			let _ = Staking::on_offence(offenders, slash_fraction, start_session).unwrap();
+			return;
 		} else if bonded_era > era {
-			break
+			break;
 		}
 	}
 
 	if Staking::active_era().unwrap().index == era {
-		Staking::on_offence(offenders, slash_fraction, Staking::eras_start_session_index(era).unwrap());
+		let _ =
+			Staking::on_offence(
+				offenders,
+				slash_fraction,
+				Staking::eras_start_session_index(era).unwrap()
+			).unwrap();
 	} else {
 		panic!("cannot slash in era {}", era);
 	}
@@ -513,6 +668,193 @@ pub fn on_offence_now(
 	on_offence_in_era(offenders, slash_fraction, now)
 }
 
+// winners will be chosen by simply their unweighted total backing stake. Nominator stake is
+// distributed evenly.
+pub fn horrible_phragmen_with_post_processing(
+	do_reduce: bool,
+) -> (CompactAssignments, Vec<ValidatorIndex>, PhragmenScore) {
+	use std::collections::BTreeMap;
+
+	let mut backing_stake_of: BTreeMap<AccountId, Balance> = BTreeMap::new();
+
+	// self stake
+	<Validators<Test>>::iter().for_each(|(who, _p)| {
+		*backing_stake_of.entry(who).or_insert(Zero::zero()) += Staking::slashable_balance_of(&who)
+	});
+
+	// add nominator stuff
+	<Nominators<Test>>::iter().for_each(|(who, nomination)| {
+		nomination.targets.iter().for_each(|v| {
+			*backing_stake_of.entry(*v).or_insert(Zero::zero()) +=
+				Staking::slashable_balance_of(&who)
+		})
+	});
+
+	// elect winners
+	let mut sorted: Vec<AccountId> = backing_stake_of.keys().cloned().collect();
+	sorted.sort_by_key(|x| backing_stake_of.get(x).unwrap());
+	let winners: Vec<AccountId> = sorted
+		.iter()
+		.cloned()
+		.take(Staking::validator_count() as usize)
+		.collect();
+
+	// create assignments
+	let mut staked_assignment: Vec<StakedAssignment<AccountId>> = Vec::new();
+	<Nominators<Test>>::iter().for_each(|(who, nomination)| {
+		let mut dist: Vec<(AccountId, ExtendedBalance)> = Vec::new();
+		nomination.targets.iter().for_each(|v| {
+			if winners.iter().find(|w| *w == v).is_some() {
+				dist.push((*v, ExtendedBalance::zero()));
+			}
+		});
+
+		if dist.len() == 0 {
+			return;
+		}
+
+		// assign real stakes. just split the stake.
+		let stake = Staking::slashable_balance_of(&who) as ExtendedBalance;
+		let mut sum: ExtendedBalance = Zero::zero();
+		let dist_len = dist.len();
+		{
+			dist.iter_mut().for_each(|(_, w)| {
+				let partial = stake / (dist_len as ExtendedBalance);
+				*w = partial;
+				sum += partial;
+			});
+		}
+
+		// assign the leftover to last.
+		{
+			let leftover = stake - sum;
+			let last = dist.last_mut().unwrap();
+			last.1 += leftover;
+		}
+
+		staked_assignment.push(StakedAssignment {
+			who,
+			distribution: dist,
+		});
+	});
+
+	// Ensure that this result is worse than seq-phragmen. Otherwise, it should not have been used
+	// for testing.
+	let score = {
+		let (_, _, better_score) = prepare_submission_with(true, |_| {});
+
+		let support = build_support_map::<AccountId>(&winners, &staked_assignment).0;
+		let score = evaluate_support(&support);
+
+		assert!(sp_phragmen::is_score_better(score, better_score));
+
+		score
+	};
+
+	if do_reduce {
+		reduce(&mut staked_assignment);
+	}
+
+	let snapshot_validators = Staking::snapshot_validators().unwrap();
+	let snapshot_nominators = Staking::snapshot_nominators().unwrap();
+	let nominator_index = |a: &AccountId| -> Option<NominatorIndex> {
+		snapshot_nominators.iter().position(|x| x == a).map(|i| i as NominatorIndex)
+	};
+	let validator_index = |a: &AccountId| -> Option<ValidatorIndex> {
+		snapshot_validators.iter().position(|x| x == a).map(|i| i as ValidatorIndex)
+	};
+
+	// convert back to ratio assignment. This takes less space.
+	let assignments_reduced =
+		sp_phragmen::assignment_staked_to_ratio::<AccountId, OffchainAccuracy>(staked_assignment);
+
+	let compact =
+		CompactAssignments::from_assignment(assignments_reduced, nominator_index, validator_index)
+			.unwrap();
+
+	// winner ids to index
+	let winners = winners.into_iter().map(|w| validator_index(&w).unwrap()).collect::<Vec<_>>();
+
+	(compact, winners, score)
+}
+
+// Note: this should always logically reproduce [`offchain_election::prepare_submission`], yet we
+// cannot do it since we want to have `tweak` injected into the process.
+pub fn prepare_submission_with(
+	do_reduce: bool,
+	tweak: impl FnOnce(&mut Vec<StakedAssignment<AccountId>>),
+) -> (CompactAssignments, Vec<ValidatorIndex>, PhragmenScore) {
+	// run phragmen on the default stuff.
+	let sp_phragmen::PhragmenResult {
+		winners,
+		assignments,
+	} = Staking::do_phragmen::<OffchainAccuracy>().unwrap();
+	let winners = winners.into_iter().map(|(w, _)| w).collect::<Vec<AccountId>>();
+
+	let stake_of = |who: &AccountId| -> ExtendedBalance {
+		<CurrencyToVoteHandler as Convert<Balance, u64>>::convert(
+			Staking::slashable_balance_of(&who)
+		) as ExtendedBalance
+	};
+	let mut staked = sp_phragmen::assignment_ratio_to_staked(assignments, stake_of);
+
+	// apply custom tweaks. awesome for testing.
+	tweak(&mut staked);
+
+	if do_reduce {
+		reduce(&mut staked);
+	}
+
+	// convert back to ratio assignment. This takes less space.
+	let snapshot_validators = Staking::snapshot_validators().expect("snapshot not created.");
+	let snapshot_nominators = Staking::snapshot_nominators().expect("snapshot not created.");
+	let nominator_index = |a: &AccountId| -> Option<NominatorIndex> {
+		snapshot_nominators
+			.iter()
+			.position(|x| x == a)
+			.map_or_else(
+				|| { println!("unable to find nominator index for {:?}", a); None },
+				|i| Some(i as NominatorIndex),
+			)
+	};
+	let validator_index = |a: &AccountId| -> Option<ValidatorIndex> {
+		snapshot_validators
+			.iter()
+			.position(|x| x == a)
+			.map_or_else(
+				|| { println!("unable to find validator index for {:?}", a); None },
+				|i| Some(i as ValidatorIndex),
+			)
+	};
+
+	let assignments_reduced = sp_phragmen::assignment_staked_to_ratio(staked);
+
+	// re-compute score by converting, yet again, into staked type
+	let score = {
+		let staked = sp_phragmen::assignment_ratio_to_staked(
+			assignments_reduced.clone(),
+			Staking::slashable_balance_of_extended,
+		);
+
+		let (support_map, _) = build_support_map::<AccountId>(
+			winners.as_slice(),
+			staked.as_slice(),
+		);
+		evaluate_support::<AccountId>(&support_map)
+	};
+
+	let compact =
+		CompactAssignments::from_assignment(assignments_reduced, nominator_index, validator_index)
+			.map_err(|e| { println!("error in compact: {:?}", e); e })
+			.expect("Failed to create compact");
+
+
+	// winner ids to index
+	let winners = winners.into_iter().map(|w| validator_index(&w).unwrap()).collect::<Vec<_>>();
+
+	(compact, winners, score)
+}
+
 /// Make all validator and nominator request their payment
 pub fn make_all_reward_payment(era: EraIndex) {
 	let validators_with_reward = ErasRewardPoints::<Test>::get(era).individual.keys()
@@ -544,3 +886,23 @@ pub fn make_all_reward_payment(era: EraIndex) {
 		assert_ok!(Staking::payout_validator(Origin::signed(validator_controller), era));
 	}
 }
+
+#[macro_export]
+macro_rules! assert_session_era {
+	($session:expr, $era:expr) => {
+		assert_eq!(
+			Session::current_index(),
+			$session,
+			"wrong session {} != {}",
+			Session::current_index(),
+			$session,
+		);
+		assert_eq!(
+			Staking::active_era().unwrap().index,
+			$era,
+			"wrong active era {} != {}",
+			Staking::active_era().unwrap().index,
+			$era,
+		);
+	};
+}

substrate/frame/staking/src/offchain_election.rs

@@ -0,0 +1,219 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helpers for offchain worker election.
+
+use crate::{
+	Call, CompactAssignments, Module, NominatorIndex, OffchainAccuracy, Trait, ValidatorIndex,
+};
+use frame_system::offchain::SubmitUnsignedTransaction;
+use sp_phragmen::{
+	build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, PhragmenResult,
+	PhragmenScore,
+};
+use sp_runtime::offchain::storage::StorageValueRef;
+use sp_runtime::PerThing;
+use sp_runtime::RuntimeDebug;
+use sp_std::{convert::TryInto, prelude::*};
+
+/// Error types related to the offchain election machinery.
+#[derive(RuntimeDebug)]
+pub enum OffchainElectionError {
+	/// Phragmen election returned None. This means less candidate that minimum number of needed
+	/// validators were present. The chain is in trouble and not much that we can do about it.
+	ElectionFailed,
+	/// Submission to the transaction pool failed.
+	PoolSubmissionFailed,
+	/// The snapshot data is not available.
+	SnapshotUnavailable,
+	/// Error from phragmen crate. This usually relates to compact operation.
+	PhragmenError(sp_phragmen::Error),
+	/// One of the computed winners is invalid.
+	InvalidWinner,
+}
+
+impl From<sp_phragmen::Error> for OffchainElectionError {
+	fn from(e: sp_phragmen::Error) -> Self {
+		Self::PhragmenError(e)
+	}
+}
+
+/// Storage key used to store the persistent offchain worker status.
+pub(crate) const OFFCHAIN_HEAD_DB: &[u8] = b"parity/staking-election/";
+/// The repeat threshold of the offchain worker. This means we won't run the offchain worker twice
+/// within a window of 5 blocks.
+pub(crate) const OFFCHAIN_REPEAT: u32 = 5;
+/// Default number of blocks for which the unsigned transaction should stay in the pool
+pub(crate) const DEFAULT_LONGEVITY: u64 = 25;
+
+/// Checks if an execution of the offchain worker is permitted at the given block number, or not.
+///
+/// This essentially makes sure that we don't run on previous blocks in case of a re-org, and we
+/// don't run twice within a window of length [`OFFCHAIN_REPEAT`].
+///
+/// Returns `Ok(())` if offchain worker should happen, `Err(reason)` otherwise.
+pub(crate) fn set_check_offchain_execution_status<T: Trait>(
+	now: T::BlockNumber,
+) -> Result<(), &'static str> {
+	let storage = StorageValueRef::persistent(&OFFCHAIN_HEAD_DB);
+	let threshold = T::BlockNumber::from(OFFCHAIN_REPEAT);
+
+	let mutate_stat =
+		storage.mutate::<_, &'static str, _>(|maybe_head: Option<Option<T::BlockNumber>>| {
+			match maybe_head {
+				Some(Some(head)) if now < head => Err("fork."),
+				Some(Some(head)) if now >= head && now <= head + threshold => {
+					Err("recently executed.")
+				}
+				Some(Some(head)) if now > head + threshold => {
+					// we can run again now. Write the new head.
+					Ok(now)
+				}
+				_ => {
+					// value doesn't exists. Probably this node just booted up. Write, and run
+					Ok(now)
+				}
+			}
+		});
+
+	match mutate_stat {
+		// all good
+		Ok(Ok(_)) => Ok(()),
+		// failed to write.
+		Ok(Err(_)) => Err("failed to write to offchain db."),
+		// fork etc.
+		Err(why) => Err(why),
+	}
+}
+
+/// The internal logic of the offchain worker of this module. This runs the phragmen election,
+/// compacts and reduces the solution, computes the score and submits it back to the chain as an
+/// unsigned transaction, without any signature.
+pub(crate) fn compute_offchain_election<T: Trait>() -> Result<(), OffchainElectionError> {
+	// compute raw solution. Note that we use `OffchainAccuracy`.
+	let PhragmenResult {
+		winners,
+		assignments,
+	} = <Module<T>>::do_phragmen::<OffchainAccuracy>()
+		.ok_or(OffchainElectionError::ElectionFailed)?;
+
+	// process and prepare it for submission.
+	let (winners, compact, score) = prepare_submission::<T>(assignments, winners, true)?;
+
+	// defensive-only: active era can never be none except genesis.
+	let era = <Module<T>>::active_era().map(|e| e.index).unwrap_or_default();
+
+	// send it.
+	let call: <T as Trait>::Call = Call::submit_election_solution_unsigned(
+		winners,
+		compact,
+		score,
+		era,
+	).into();
+
+	T::SubmitTransaction::submit_unsigned(call)
+		.map_err(|_| OffchainElectionError::PoolSubmissionFailed)
+}
+
+/// Takes a phragmen result and spits out some data that can be submitted to the chain.
+///
+/// This does a lot of stuff; read the inline comments.
+pub fn prepare_submission<T: Trait>(
+	assignments: Vec<Assignment<T::AccountId, OffchainAccuracy>>,
+	winners: Vec<(T::AccountId, ExtendedBalance)>,
+	do_reduce: bool,
+) -> Result<(Vec<ValidatorIndex>, CompactAssignments, PhragmenScore), OffchainElectionError> where
+	ExtendedBalance: From<<OffchainAccuracy as PerThing>::Inner>,
+{
+	// make sure that the snapshot is available.
+	let snapshot_validators =
+		<Module<T>>::snapshot_validators().ok_or(OffchainElectionError::SnapshotUnavailable)?;
+	let snapshot_nominators =
+		<Module<T>>::snapshot_nominators().ok_or(OffchainElectionError::SnapshotUnavailable)?;
+
+	// all helper closures
+	let nominator_index = |a: &T::AccountId| -> Option<NominatorIndex> {
+		snapshot_nominators
+			.iter()
+			.position(|x| x == a)
+			.and_then(|i| <usize as TryInto<NominatorIndex>>::try_into(i).ok())
+	};
+	let validator_index = |a: &T::AccountId| -> Option<ValidatorIndex> {
+		snapshot_validators
+			.iter()
+			.position(|x| x == a)
+			.and_then(|i| <usize as TryInto<ValidatorIndex>>::try_into(i).ok())
+	};
+
+	// Clean winners.
+	let winners = winners
+		.into_iter()
+		.map(|(w, _)| w)
+		.collect::<Vec<T::AccountId>>();
+
+	// convert into absolute value and to obtain the reduced version.
+	let mut staked = sp_phragmen::assignment_ratio_to_staked(
+		assignments,
+		<Module<T>>::slashable_balance_of_extended,
+	);
+
+	if do_reduce {
+		reduce(&mut staked);
+	}
+
+	// Convert back to ratio assignment. This takes less space.
+	let low_accuracy_assignment = sp_phragmen::assignment_staked_to_ratio(staked);
+
+	// convert back to staked to compute the score in the receiver's accuracy. This can be done
+	// nicer, for now we do it as such since this code is not time-critical. This ensure that the
+	// score _predicted_ here is the same as the one computed on chain and you will not get a
+	// `PhragmenBogusScore` error. This is totally NOT needed if we don't do reduce. This whole
+	// _accuracy glitch_ happens because reduce breaks that assumption of rounding and **scale**.
+	// The initial phragmen results are computed in `OffchainAccuracy` and the initial `staked`
+	// assignment set is also all multiples of this value. After reduce, this no longer holds. Hence
+	// converting to ratio thereafter is not trivially reversible.
+	let score = {
+		let staked = sp_phragmen::assignment_ratio_to_staked(
+			low_accuracy_assignment.clone(),
+			<Module<T>>::slashable_balance_of_extended,
+		);
+
+		let (support_map, _) = build_support_map::<T::AccountId>(&winners, &staked);
+		evaluate_support::<T::AccountId>(&support_map)
+	};
+
+	// compact encode the assignment.
+	let compact = CompactAssignments::from_assignment(
+		low_accuracy_assignment,
+		nominator_index,
+		validator_index,
+	).map_err(|e| OffchainElectionError::from(e))?;
+
+	// winners to index. Use a simple for loop for a more expressive early exit in case of error.
+	let mut winners_indexed: Vec<ValidatorIndex> = Vec::with_capacity(winners.len());
+	for w in winners {
+		if let Some(idx) = snapshot_validators.iter().position(|v| *v == w) {
+			let compact_index: ValidatorIndex = idx
+				.try_into()
+				.map_err(|_| OffchainElectionError::InvalidWinner)?;
+			winners_indexed.push(compact_index);
+		} else {
+			return Err(OffchainElectionError::InvalidWinner);
+		}
+	}
+
+	Ok((winners_indexed, compact, score))
+}

substrate/frame/staking/src/slashing.rs

@@ -16,11 +16,11 @@
 
 //! A slashing implementation for NPoS systems.
 //!
-//! For the purposes of the economic model, it is easiest to think of each validator
-//! of a nominator which nominates only its own identity.
+//! For the purposes of the economic model, it is easiest to think of each validator as a nominator
+//! which nominates only its own identity.
 //!
-//! The act of nomination signals intent to unify economic identity with the validator - to take part in the
-//! rewards of a job well done, and to take part in the punishment of a job done badly.
+//! The act of nomination signals intent to unify economic identity with the validator - to take
+//! part in the rewards of a job well done, and to take part in the punishment of a job done badly.
 //!
 //! There are 3 main difficulties to account for with slashing in NPoS:
 //!   - A nominator can nominate multiple validators and be slashed via any of them.
@@ -52,7 +52,7 @@ use super::{
 	EraIndex, Trait, Module, Store, BalanceOf, Exposure, Perbill, SessionInterface,
 	NegativeImbalanceOf, UnappliedSlash,
 };
-use sp_runtime::{traits::{Zero, Saturating}, PerThing};
+use sp_runtime::{traits::{Zero, Saturating}, PerThing, RuntimeDebug};
 use frame_support::{
 	StorageMap, StorageDoubleMap,
 	traits::{Currency, OnUnbalanced, Imbalance},
@@ -65,7 +65,7 @@ use codec::{Encode, Decode};
 const REWARD_F1: Perbill = Perbill::from_percent(50);
 
 /// The index of a slashing span - unique to each stash.
-pub(crate) type SpanIndex = u32;
+pub type SpanIndex = u32;
 
 // A range of start..end eras for a slashing span.
 #[derive(Encode, Decode)]
@@ -83,7 +83,7 @@ impl SlashingSpan {
 }
 
 /// An encoding of all of a nominator's slashing spans.
-#[derive(Encode, Decode)]
+#[derive(Encode, Decode, RuntimeDebug)]
 pub struct SlashingSpans {
 	// the index of the current slashing span of the nominator. different for
 	// every stash, resets when the account hits free balance 0.
@@ -143,7 +143,7 @@ impl SlashingSpans {
 	}
 
 	/// Yields the era index where the most recent non-zero slash occurred.
-	pub(crate) fn last_nonzero_slash(&self) -> EraIndex {
+	pub fn last_nonzero_slash(&self) -> EraIndex {
 		self.last_nonzero_slash
 	}
 

substrate/frame/staking/src/testing_utils.rs

@@ -0,0 +1,340 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Testing utils for staking. Needs the `testing-utils` feature to be enabled.
+//!
+//! Note that these helpers should NOT be used with the actual crate tests, but are rather designed
+//! for when the module is being externally tested (i.e. fuzzing, benchmarking, e2e tests). Enabling
+//! this feature in the current crate's Cargo.toml will leak the all of this into a normal release
+//! build. Just don't do it.
+
+use crate::*;
+use codec::{Decode, Encode};
+use frame_support::assert_ok;
+use frame_system::RawOrigin;
+use pallet_indices::address::Address;
+use rand::Rng;
+use sp_core::hashing::blake2_256;
+use sp_phragmen::{
+	build_support_map, evaluate_support, reduce, Assignment, PhragmenScore, StakedAssignment,
+};
+
+const CTRL_PREFIX: u32 = 1000;
+const NOMINATOR_PREFIX: u32 = 1_000_000;
+
+/// A dummy suer.
+pub const USER: u32 = 999_999_999;
+
+/// Address type of the `T`
+pub type AddressOf<T> = Address<<T as frame_system::Trait>::AccountId, u32>;
+
+/// Random number in the range `[a, b]`.
+pub fn random(a: u32, b: u32) -> u32 {
+	rand::thread_rng().gen_range(a, b)
+}
+
+/// Set the desired validator count, with related storage items.
+pub fn set_validator_count<T: Trait>(to_elect: u32) {
+	ValidatorCount::put(to_elect);
+	MinimumValidatorCount::put(to_elect / 2);
+	<EraElectionStatus<T>>::put(ElectionStatus::Open(T::BlockNumber::from(1u32)));
+}
+
+/// Build an account with the given index.
+pub fn account<T: Trait>(index: u32) -> T::AccountId {
+	let entropy = (b"benchmark/staking", index).using_encoded(blake2_256);
+	T::AccountId::decode(&mut &entropy[..]).unwrap_or_default()
+}
+
+/// Build an address given Index
+pub fn address<T: Trait>(index: u32) -> AddressOf<T> {
+	pallet_indices::address::Address::Id(account::<T>(index))
+}
+
+/// Generate signed origin from `who`.
+pub fn signed<T: Trait>(who: T::AccountId) -> T::Origin {
+	RawOrigin::Signed(who).into()
+}
+
+/// Generate signed origin from `index`.
+pub fn signed_account<T: Trait>(index: u32) -> T::Origin {
+	signed::<T>(account::<T>(index))
+}
+
+/// Bond a validator.
+pub fn bond_validator<T: Trait>(stash: T::AccountId, ctrl: u32, val: BalanceOf<T>)
+where
+	T::Lookup: StaticLookup<Source = AddressOf<T>>,
+{
+	let _ = T::Currency::make_free_balance_be(&stash, val);
+	assert_ok!(<Module<T>>::bond(
+		signed::<T>(stash),
+		address::<T>(ctrl),
+		val,
+		RewardDestination::Controller
+	));
+	assert_ok!(<Module<T>>::validate(
+		signed_account::<T>(ctrl),
+		ValidatorPrefs::default()
+	));
+}
+
+pub fn bond_nominator<T: Trait>(
+	stash: T::AccountId,
+	ctrl: u32,
+	val: BalanceOf<T>,
+	target: Vec<AddressOf<T>>,
+) where
+	T::Lookup: StaticLookup<Source = AddressOf<T>>,
+{
+	let _ = T::Currency::make_free_balance_be(&stash, val);
+	assert_ok!(<Module<T>>::bond(
+		signed::<T>(stash),
+		address::<T>(ctrl),
+		val,
+		RewardDestination::Controller
+	));
+	assert_ok!(<Module<T>>::nominate(signed_account::<T>(ctrl), target));
+}
+
+/// Bond `nun_validators` validators and `num_nominator` nominators with `edge_per_voter` random
+/// votes per nominator.
+pub fn setup_chain_stakers<T: Trait>(num_validators: u32, num_voters: u32, edge_per_voter: u32)
+where
+	T::Lookup: StaticLookup<Source = AddressOf<T>>,
+{
+	(0..num_validators).for_each(|i| {
+		bond_validator::<T>(
+			account::<T>(i),
+			i + CTRL_PREFIX,
+			<BalanceOf<T>>::from(random(1, 1000)) * T::Currency::minimum_balance(),
+		);
+	});
+
+	(0..num_voters).for_each(|i| {
+		let mut targets: Vec<AddressOf<T>> = Vec::with_capacity(edge_per_voter as usize);
+		let mut all_targets = (0..num_validators)
+			.map(|t| address::<T>(t))
+			.collect::<Vec<_>>();
+		assert!(num_validators >= edge_per_voter);
+		(0..edge_per_voter).for_each(|_| {
+			let target = all_targets.remove(random(0, all_targets.len() as u32 - 1) as usize);
+			targets.push(target);
+		});
+		bond_nominator::<T>(
+			account::<T>(i + NOMINATOR_PREFIX),
+			i + NOMINATOR_PREFIX + CTRL_PREFIX,
+			<BalanceOf<T>>::from(random(1, 1000)) * T::Currency::minimum_balance(),
+			targets,
+		);
+	});
+
+	<Module<T>>::create_stakers_snapshot();
+}
+
+/// Build a _really bad_ but acceptable solution for election. This should always yield a solution
+/// which has a less score than the seq-phragmen.
+pub fn get_weak_solution<T: Trait>(
+	do_reduce: bool,
+) -> (Vec<ValidatorIndex>, CompactAssignments, PhragmenScore) {
+	let mut backing_stake_of: BTreeMap<T::AccountId, BalanceOf<T>> = BTreeMap::new();
+
+	// self stake
+	<Validators<T>>::enumerate().for_each(|(who, _p)| {
+		*backing_stake_of.entry(who.clone()).or_insert(Zero::zero()) +=
+			<Module<T>>::slashable_balance_of(&who)
+	});
+
+	// add nominator stuff
+	<Nominators<T>>::enumerate().for_each(|(who, nomination)| {
+		nomination.targets.into_iter().for_each(|v| {
+			*backing_stake_of.entry(v).or_insert(Zero::zero()) +=
+				<Module<T>>::slashable_balance_of(&who)
+		})
+	});
+
+	// elect winners
+	let mut sorted: Vec<T::AccountId> = backing_stake_of.keys().cloned().collect();
+	sorted.sort_by_key(|x| backing_stake_of.get(x).unwrap());
+	let winners: Vec<T::AccountId> = sorted
+		.iter()
+		.cloned()
+		.take(<Module<T>>::validator_count() as usize)
+		.collect();
+
+	let mut staked_assignments: Vec<StakedAssignment<T::AccountId>> = Vec::new();
+	<Nominators<T>>::enumerate().for_each(|(who, nomination)| {
+		let mut dist: Vec<(T::AccountId, ExtendedBalance)> = Vec::new();
+		nomination.targets.into_iter().for_each(|v| {
+			if winners.iter().find(|&w| *w == v).is_some() {
+				dist.push((v, ExtendedBalance::zero()));
+			}
+		});
+
+		if dist.len() == 0 {
+			return;
+		}
+
+		// assign real stakes. just split the stake.
+		let stake = <T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(
+			<Module<T>>::slashable_balance_of(&who),
+		) as ExtendedBalance;
+
+		let mut sum: ExtendedBalance = Zero::zero();
+		let dist_len = dist.len() as ExtendedBalance;
+
+		// assign main portion
+		// only take the first half into account. This should highly imbalance stuff, which is good.
+		dist.iter_mut()
+			.take(if dist_len > 1 {
+				(dist_len as usize) / 2
+			} else {
+				1
+			})
+			.for_each(|(_, w)| {
+				let partial = stake / dist_len;
+				*w = partial;
+				sum += partial;
+			});
+
+		// assign the leftover to last.
+		let leftover = stake - sum;
+		let last = dist.last_mut().unwrap();
+		last.1 += leftover;
+
+		staked_assignments.push(StakedAssignment {
+			who,
+			distribution: dist,
+		});
+	});
+
+	// add self support to winners.
+	winners.iter().for_each(|w| {
+		staked_assignments.push(StakedAssignment {
+			who: w.clone(),
+			distribution: vec![(
+				w.clone(),
+				<T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(
+					<Module<T>>::slashable_balance_of(&w),
+				) as ExtendedBalance,
+			)],
+		})
+	});
+
+	if do_reduce {
+		reduce(&mut staked_assignments);
+	}
+
+	// helpers for building the compact
+	let snapshot_validators = <Module<T>>::snapshot_validators().unwrap();
+	let snapshot_nominators = <Module<T>>::snapshot_nominators().unwrap();
+
+	let nominator_index = |a: &T::AccountId| -> Option<NominatorIndex> {
+		snapshot_nominators
+			.iter()
+			.position(|x| x == a)
+			.and_then(|i| <usize as TryInto<NominatorIndex>>::try_into(i).ok())
+	};
+	let validator_index = |a: &T::AccountId| -> Option<ValidatorIndex> {
+		snapshot_validators
+			.iter()
+			.position(|x| x == a)
+			.and_then(|i| <usize as TryInto<ValidatorIndex>>::try_into(i).ok())
+	};
+	let stake_of = |who: &T::AccountId| -> ExtendedBalance {
+		<T::CurrencyToVote as Convert<BalanceOf<T>, u64>>::convert(
+			<Module<T>>::slashable_balance_of(who),
+		) as ExtendedBalance
+	};
+
+	// convert back to ratio assignment. This takes less space.
+	let low_accuracy_assignment: Vec<Assignment<T::AccountId, OffchainAccuracy>> =
+		staked_assignments
+			.into_iter()
+			.map(|sa| sa.into_assignment(true))
+			.collect();
+
+	// re-calculate score based on what the chain will decode.
+	let score = {
+		let staked: Vec<StakedAssignment<T::AccountId>> = low_accuracy_assignment
+			.iter()
+			.map(|a| {
+				let stake = stake_of(&a.who);
+				a.clone().into_staked(stake, true)
+			})
+			.collect();
+
+		let (support_map, _) =
+			build_support_map::<T::AccountId>(winners.as_slice(), staked.as_slice());
+		evaluate_support::<T::AccountId>(&support_map)
+	};
+
+	// compact encode the assignment.
+	let compact = CompactAssignments::from_assignment(
+		low_accuracy_assignment,
+		nominator_index,
+		validator_index,
+	)
+	.unwrap();
+
+	// winners to index.
+	let winners = winners
+		.into_iter()
+		.map(|w| {
+			snapshot_validators
+				.iter()
+				.position(|v| *v == w)
+				.unwrap()
+				.try_into()
+				.unwrap()
+		})
+		.collect::<Vec<ValidatorIndex>>();
+
+	(winners, compact, score)
+}
+
+/// Create a solution for seq-phragmen. This uses the same internal function as used by the offchain
+/// worker code.
+pub fn get_seq_phragmen_solution<T: Trait>(
+	do_reduce: bool,
+) -> (Vec<ValidatorIndex>, CompactAssignments, PhragmenScore) {
+	let sp_phragmen::PhragmenResult {
+		winners,
+		assignments,
+	} = <Module<T>>::do_phragmen::<OffchainAccuracy>().unwrap();
+
+	offchain_election::prepare_submission::<T>(assignments, winners, do_reduce).unwrap()
+}
+
+/// Remove all validator, nominators, votes and exposures.
+pub fn clean<T: Trait>(era: EraIndex)
+	where
+		<T as frame_system::Trait>::AccountId: codec::EncodeLike<u32>,
+		u32: codec::EncodeLike<T::AccountId>,
+{
+	<Validators<T>>::enumerate().for_each(|(k, _)| {
+		let ctrl = <Module<T>>::bonded(&k).unwrap();
+		<Bonded<T>>::remove(&k);
+		<Validators<T>>::remove(&k);
+		<Ledger<T>>::remove(&ctrl);
+		<ErasStakers<T>>::remove(k, era);
+	});
+	<Nominators<T>>::enumerate().for_each(|(k, _)| <Nominators<T>>::remove(k));
+	<Ledger<T>>::remove_all();
+	<Bonded<T>>::remove_all();
+	<QueuedElected<T>>::kill();
+	QueuedScore::kill();
+}

substrate/frame/support/src/traits.rs

@@ -24,7 +24,7 @@ use sp_core::u32_trait::Value as U32;
 use sp_runtime::{
 	RuntimeDebug,
 	ConsensusEngineId, DispatchResult, DispatchError,
-	traits::{MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput},
+	traits::{MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput, Bounded},
 };
 use crate::dispatch::Parameter;
 use crate::storage::StorageMap;
@@ -87,7 +87,7 @@ impl<
 	Created: Happened<K>,
 	Removed: Happened<K>,
 	K: FullCodec,
-	T: FullCodec
+	T: FullCodec,
 > StoredMap<K, T> for StorageMapShim<S, Created, Removed, K, T> {
 	fn get(k: &K) -> T { S::get(k) }
 	fn is_explicit(k: &K) -> bool { S::contains_key(k) }
@@ -138,6 +138,35 @@ impl<
 	}
 }
 
+/// Something that can estimate at which block the next session rotation will happen. This should
+/// be the same logical unit that dictates `ShouldEndSession` to the session module. No Assumptions
+/// are made about the scheduling of the sessions.
+pub trait EstimateNextSessionRotation<BlockNumber> {
+	/// Return the block number at which the next session rotation is estimated to happen.
+	///
+	/// None should be returned if the estimation fails to come to an answer
+	fn estimate_next_session_rotation(now: BlockNumber) -> Option<BlockNumber>;
+}
+
+impl<BlockNumber: Bounded> EstimateNextSessionRotation<BlockNumber> for () {
+	fn estimate_next_session_rotation(_: BlockNumber) -> Option<BlockNumber> {
+		Default::default()
+	}
+}
+
+/// Something that can estimate at which block the next `new_session` will be triggered. This must
+/// always be implemented by the session module.
+pub trait EstimateNextNewSession<BlockNumber> {
+	/// Return the block number at which the next new session is estimated to happen.
+	fn estimate_next_new_session(now: BlockNumber) -> Option<BlockNumber>;
+}
+
+impl<BlockNumber: Bounded> EstimateNextNewSession<BlockNumber> for () {
+	fn estimate_next_new_session(_: BlockNumber) -> Option<BlockNumber> {
+		Default::default()
+	}
+}
+
 /// Anything that can have a `::len()` method.
 pub trait Len {
 	/// Return the length of data type.

substrate/frame/system/src/offchain.rs

@@ -95,7 +95,8 @@ impl<Public, Signature, TAnyAppPublic> Signer<Public, Signature> for TAnyAppPubl
 }
 
 /// Retrieves a public key type for given `SignAndSubmitTransaction`.
-pub type PublicOf<T, Call, X> = <
+pub type PublicOf<T, Call, X> =
+<
 	<X as SignAndSubmitTransaction<T, Call>>::CreateTransaction
 	as
 	CreateTransaction<T, <X as SignAndSubmitTransaction<T, Call>>::Extrinsic>
@@ -109,7 +110,7 @@ pub type PublicOf<T, Call, X> = <
 /// you should use.
 pub trait SignAndSubmitTransaction<T: crate::Trait, Call> {
 	/// Unchecked extrinsic type.
-	type Extrinsic: ExtrinsicT<Call=Call> + codec::Encode;
+	type Extrinsic: ExtrinsicT<Call=Call> + Encode;
 
 	/// A runtime-specific type to produce signed data for the extrinsic.
 	type CreateTransaction: CreateTransaction<T, Self::Extrinsic>;
@@ -156,7 +157,7 @@ pub trait SignAndSubmitTransaction<T: crate::Trait, Call> {
 /// you should use.
 pub trait SubmitUnsignedTransaction<T: crate::Trait, Call> {
 	/// Unchecked extrinsic type.
-	type Extrinsic: ExtrinsicT<Call=Call> + codec::Encode;
+	type Extrinsic: ExtrinsicT<Call=Call> + Encode;
 
 	/// Submit given call to the transaction pool as unsigned transaction.
 	///
@@ -164,7 +165,8 @@ pub trait SubmitUnsignedTransaction<T: crate::Trait, Call> {
 	/// and `Err` if transaction was rejected from the pool.
 	fn submit_unsigned(call: impl Into<Call>) -> Result<(), ()> {
 		let xt = Self::Extrinsic::new(call.into(), None).ok_or(())?;
-		sp_io::offchain::submit_transaction(xt.encode())
+		let encoded_xt = xt.encode();
+		sp_io::offchain::submit_transaction(encoded_xt)
 	}
 }
 
@@ -291,7 +293,7 @@ impl<T, E, S, C, Call> SignAndSubmitTransaction<T, Call> for TransactionSubmitte
 	T: crate::Trait,
 	C: CreateTransaction<T, E>,
 	S: Signer<<C as CreateTransaction<T, E>>::Public, <C as CreateTransaction<T, E>>::Signature>,
-	E: ExtrinsicT<Call=Call> + codec::Encode,
+	E: ExtrinsicT<Call=Call> + Encode,
 {
 	type Extrinsic = E;
 	type CreateTransaction = C;
@@ -301,7 +303,7 @@ impl<T, E, S, C, Call> SignAndSubmitTransaction<T, Call> for TransactionSubmitte
 /// A blanket implementation to use the same submitter for unsigned transactions as well.
 impl<T, E, S, C, Call> SubmitUnsignedTransaction<T, Call> for TransactionSubmitter<S, C, E> where
 	T: crate::Trait,
-	E: ExtrinsicT<Call=Call> + codec::Encode,
+	E: ExtrinsicT<Call=Call> + Encode,
 {
 	type Extrinsic = E;
 }
@@ -310,7 +312,7 @@ impl<T, E, S, C, Call> SubmitUnsignedTransaction<T, Call> for TransactionSubmitt
 impl<T, C, E, S, Call> SubmitSignedTransaction<T, Call> for TransactionSubmitter<S, C, E> where
 	T: crate::Trait,
 	C: CreateTransaction<T, E>,
-	E: ExtrinsicT<Call=Call> + codec::Encode,
+	E: ExtrinsicT<Call=Call> + Encode,
 	S: Signer<<C as CreateTransaction<T, E>>::Public, <C as CreateTransaction<T, E>>::Signature>,
 	// Make sure we can unwrap the app crypto key.
 	S: RuntimeAppPublic + AppPublic + Into<<S as AppPublic>::Generic>,

substrate/frame/transaction-payment/src/lib.rs

@@ -106,11 +106,6 @@ impl<T: Trait> Module<T> {
 	///
 	/// All dispatchables must be annotated with weight and will have some fee info. This function
 	/// always returns.
-	// NOTE: we can actually make it understand `ChargeTransactionPayment`, but would be some hassle
-	// for sure. We have to make it aware of the index of `ChargeTransactionPayment` in `Extra`.
-	// Alternatively, we could actually execute the tx's per-dispatch and record the balance of the
-	// sender before and after the pipeline.. but this is way too much hassle for a very very little
-	// potential gain in the future.
 	pub fn query_info<Extrinsic: GetDispatchInfo>(
 		unchecked_extrinsic: Extrinsic,
 		len: u32,
@@ -119,6 +114,11 @@ impl<T: Trait> Module<T> {
 		T: Send + Sync,
 		BalanceOf<T>: Send + Sync,
 	{
+		// NOTE: we can actually make it understand `ChargeTransactionPayment`, but would be some
+		// hassle for sure. We have to make it aware of the index of `ChargeTransactionPayment` in
+		// `Extra`. Alternatively, we could actually execute the tx's per-dispatch and record the
+		// balance of the sender before and after the pipeline.. but this is way too much hassle for
+		// a very very little potential gain in the future.
 		let dispatch_info = <Extrinsic as GetDispatchInfo>::get_dispatch_info(&unchecked_extrinsic);
 
 		let partial_fee =