Extrinsic to restore corrupt staking ledgers (#3706)

This PR adds a new extrinsic `Call::restore_ledger ` gated by
`StakingAdmin` origin that restores a corrupted staking ledger. This
extrinsic will be used to recover ledgers that were affected by the
issue discussed in
https://github.com/paritytech/polkadot-sdk/issues/3245.

The extrinsic will re-write the storage items associated with a stash
account provided as input parameter. The data used to reset the ledger
can be either i) fetched on-chain or ii) partially/totally set by the
input parameters of the call.

In order to use on-chain data to restore the staking locks, we need a
way to read the current lock in the balances pallet. This PR adds a
`InspectLockableCurrency` trait and implements it in the pallet
balances. An alternative would be to tightly couple staking with the
pallet balances but that's inelegant (an example of how it would look
like in [this
branch](https://github.com/paritytech/polkadot-sdk/tree/gpestana/ledger-badstate-clean_tightly)).

More details on the type of corruptions and corresponding fixes
https://hackmd.io/DLb5jEYWSmmvqXC9ae4yRg?view#/

We verified that the `Call::restore_ledger` does fix all current
corrupted ledgers in Polkadot and Kusama. You can verify it here
https://hackmd.io/v-XNrEoGRpe7APR-EZGhOA.

**Changes introduced**
- Adds `Call::restore_ledger ` extrinsic to recover a corrupted ledger;
- Adds trait `frame_support::traits::currency::InspectLockableCurrency`
to allow external pallets to read current locks given an account and
lock ID;
- Implements the `InspectLockableCurrency` in the pallet-balances.
- Adds staking locks try-runtime checks
(https://github.com/paritytech/polkadot-sdk/issues/3751)

**Todo**
- [x] benchmark `Call::restore_ledger`
- [x] throughout testing of all ledger recovering cases
- [x] consider adding the staking locks try-runtime checks to this PR
(https://github.com/paritytech/polkadot-sdk/issues/3751)
- [x] simulate restoring all ledgers
(https://hackmd.io/Dsa2tvhISNSs7zcqriTaxQ?view) in Polkadot and Kusama
using chopsticks -- https://hackmd.io/v-XNrEoGRpe7APR-EZGhOA

Related to https://github.com/paritytech/polkadot-sdk/issues/3245
Closes https://github.com/paritytech/polkadot-sdk/issues/3751

---------

Co-authored-by: command-bot <>

substrate/frame/staking/src/benchmarking.rs

@@ -953,6 +953,15 @@ benchmarks! {
 		assert_eq!(MinCommission::<T>::get(), Perbill::from_percent(100));
 	}
 
+	restore_ledger {
+		let (stash, controller) = create_stash_controller::<T>(0, 100, RewardDestination::Staked)?;
+		// corrupt ledger.
+		Ledger::<T>::remove(controller);
+	}: _(RawOrigin::Root, stash.clone(), None, None, None)
+	verify {
+		assert_eq!(Staking::<T>::inspect_bond_state(&stash), Ok(LedgerIntegrityState::Ok));
+	}
+
 	impl_benchmark_test_suite!(
 		Staking,
 		crate::mock::ExtBuilder::default().has_stakers(true),

substrate/frame/staking/src/lib.rs

@@ -491,6 +491,20 @@ pub struct StakingLedger<T: Config> {
 	controller: Option<T::AccountId>,
 }
 
+/// State of a ledger with regards with its data and metadata integrity.
+#[derive(PartialEq, Debug)]
+enum LedgerIntegrityState {
+	/// Ledger, bond and corresponding staking lock is OK.
+	Ok,
+	/// Ledger and/or bond is corrupted. This means that the bond has a ledger with a different
+	/// stash than the bonded stash.
+	Corrupted,
+	/// Ledger was corrupted and it has been killed.
+	CorruptedKilled,
+	/// Ledger and bond are OK, however the ledger's stash lock is out of sync.
+	LockCorrupted,
+}
+
 impl<T: Config> StakingLedger<T> {
 	/// Remove entries from `unlocking` that are sufficiently old and reduce the
 	/// total by the sum of their balances.

substrate/frame/staking/src/mock.rs

@@ -25,8 +25,8 @@ use frame_election_provider_support::{
 use frame_support::{
 	assert_ok, derive_impl, ord_parameter_types, parameter_types,
 	traits::{
-		ConstU64, Currency, EitherOfDiverse, FindAuthor, Get, Hooks, Imbalance, OnUnbalanced,
-		OneSessionHandler,
+		ConstU64, Currency, EitherOfDiverse, FindAuthor, Get, Hooks, Imbalance, LockableCurrency,
+		OnUnbalanced, OneSessionHandler, WithdrawReasons,
 	},
 	weights::constants::RocksDbWeight,
 };
@@ -786,55 +786,86 @@ pub(crate) fn bond_controller_stash(controller: AccountId, stash: AccountId) ->
 	Ok(())
 }
 
+// simulates `set_controller` without corrupted ledger checks for testing purposes.
+pub(crate) fn set_controller_no_checks(stash: &AccountId) {
+	let controller = Bonded::<Test>::get(stash).expect("testing stash should be bonded");
+	let ledger = Ledger::<Test>::get(&controller).expect("testing ledger should exist");
+
+	Ledger::<Test>::remove(&controller);
+	Ledger::<Test>::insert(stash, ledger);
+	Bonded::<Test>::insert(stash, stash);
+}
+
+// simulates `bond_extra` without corrupted ledger checks for testing purposes.
+pub(crate) fn bond_extra_no_checks(stash: &AccountId, amount: Balance) {
+	let controller = Bonded::<Test>::get(stash).expect("bond must exist to bond_extra");
+	let mut ledger = Ledger::<Test>::get(&controller).expect("ledger must exist to bond_extra");
+
+	let new_total = ledger.total + amount;
+	Balances::set_lock(crate::STAKING_ID, stash, new_total, WithdrawReasons::all());
+	ledger.total = new_total;
+	ledger.active = new_total;
+	Ledger::<Test>::insert(controller, ledger);
+}
+
 pub(crate) fn setup_double_bonded_ledgers() {
-	assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 10, RewardDestination::Staked));
-	assert_ok!(Staking::bond(RuntimeOrigin::signed(2), 20, RewardDestination::Staked));
-	assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 20, RewardDestination::Staked));
+	let init_ledgers = Ledger::<Test>::iter().count();
+
+	let _ = Balances::make_free_balance_be(&333, 2000);
+	let _ = Balances::make_free_balance_be(&444, 2000);
+	let _ = Balances::make_free_balance_be(&555, 2000);
+	let _ = Balances::make_free_balance_be(&777, 2000);
+
+	assert_ok!(Staking::bond(RuntimeOrigin::signed(333), 10, RewardDestination::Staked));
+	assert_ok!(Staking::bond(RuntimeOrigin::signed(444), 20, RewardDestination::Staked));
+	assert_ok!(Staking::bond(RuntimeOrigin::signed(555), 20, RewardDestination::Staked));
 	// not relevant to the test case, but ensures try-runtime checks pass.
-	[1, 2, 3]
+	[333, 444, 555]
 		.iter()
 		.for_each(|s| Payee::<Test>::insert(s, RewardDestination::Staked));
 
 	// we want to test the case where a controller can also be a stash of another ledger.
 	// for that, we change the controller/stash bonding so that:
-	// * 2 becomes controller of 1.
-	// * 3 becomes controller of 2.
-	// * 4 becomes controller of 3.
-	let ledger_1 = Ledger::<Test>::get(1).unwrap();
-	let ledger_2 = Ledger::<Test>::get(2).unwrap();
-	let ledger_3 = Ledger::<Test>::get(3).unwrap();
+	// * 444 becomes controller of 333.
+	// * 555 becomes controller of 444.
+	// * 777 becomes controller of 555.
+	let ledger_333 = Ledger::<Test>::get(333).unwrap();
+	let ledger_444 = Ledger::<Test>::get(444).unwrap();
+	let ledger_555 = Ledger::<Test>::get(555).unwrap();
 
-	// 4 becomes controller of 3.
-	Bonded::<Test>::mutate(3, |controller| *controller = Some(4));
-	Ledger::<Test>::insert(4, ledger_3);
+	// 777 becomes controller of 555.
+	Bonded::<Test>::mutate(555, |controller| *controller = Some(777));
+	Ledger::<Test>::insert(777, ledger_555);
 
-	// 3 becomes controller of 2.
-	Bonded::<Test>::mutate(2, |controller| *controller = Some(3));
-	Ledger::<Test>::insert(3, ledger_2);
+	// 555 becomes controller of 444.
+	Bonded::<Test>::mutate(444, |controller| *controller = Some(555));
+	Ledger::<Test>::insert(555, ledger_444);
 
-	// 2 becomes controller of 1
-	Bonded::<Test>::mutate(1, |controller| *controller = Some(2));
-	Ledger::<Test>::insert(2, ledger_1);
-	// 1 is not controller anymore.
-	Ledger::<Test>::remove(1);
+	// 444 becomes controller of 333.
+	Bonded::<Test>::mutate(333, |controller| *controller = Some(444));
+	Ledger::<Test>::insert(444, ledger_333);
+
+	// 333 is not controller anymore.
+	Ledger::<Test>::remove(333);
 
 	// checks. now we have:
-	// * 3 ledgers
-	assert_eq!(Ledger::<Test>::iter().count(), 3);
-	// * stash 1 has controller 2.
-	assert_eq!(Bonded::<Test>::get(1), Some(2));
-	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(1)), Some(2));
-	assert_eq!(Ledger::<Test>::get(2).unwrap().stash, 1);
+	// * +3 ledgers
+	assert_eq!(Ledger::<Test>::iter().count(), 3 + init_ledgers);
 
-	// * stash 2 has controller 3.
-	assert_eq!(Bonded::<Test>::get(2), Some(3));
-	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(2)), Some(3));
-	assert_eq!(Ledger::<Test>::get(3).unwrap().stash, 2);
+	// * stash 333 has controller 444.
+	assert_eq!(Bonded::<Test>::get(333), Some(444));
+	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(333)), Some(444));
+	assert_eq!(Ledger::<Test>::get(444).unwrap().stash, 333);
 
-	// * stash 3 has controller 4.
-	assert_eq!(Bonded::<Test>::get(3), Some(4));
-	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(3)), Some(4));
-	assert_eq!(Ledger::<Test>::get(4).unwrap().stash, 3);
+	// * stash 444 has controller 555.
+	assert_eq!(Bonded::<Test>::get(444), Some(555));
+	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(444)), Some(555));
+	assert_eq!(Ledger::<Test>::get(555).unwrap().stash, 444);
+
+	// * stash 555 has controller 777.
+	assert_eq!(Bonded::<Test>::get(555), Some(777));
+	assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(555)), Some(777));
+	assert_eq!(Ledger::<Test>::get(777).unwrap().stash, 555);
 }
 
 #[macro_export]

substrate/frame/staking/src/pallet/impls.rs

@@ -27,8 +27,8 @@ use frame_support::{
 	dispatch::WithPostDispatchInfo,
 	pallet_prelude::*,
 	traits::{
-		Currency, Defensive, DefensiveSaturating, EstimateNextNewSession, Get, Imbalance, Len,
-		OnUnbalanced, TryCollect, UnixTime,
+		Currency, Defensive, DefensiveSaturating, EstimateNextNewSession, Get, Imbalance,
+		InspectLockableCurrency, Len, OnUnbalanced, TryCollect, UnixTime,
 	},
 	weights::Weight,
 };
@@ -50,8 +50,8 @@ use sp_std::prelude::*;
 use crate::{
 	election_size_tracker::StaticTracker, log, slashing, weights::WeightInfo, ActiveEraInfo,
 	BalanceOf, EraInfo, EraPayout, Exposure, ExposureOf, Forcing, IndividualExposure,
-	MaxNominationsOf, MaxWinnersOf, Nominations, NominationsQuota, PositiveImbalanceOf,
-	RewardDestination, SessionInterface, StakingLedger, ValidatorPrefs,
+	LedgerIntegrityState, MaxNominationsOf, MaxWinnersOf, Nominations, NominationsQuota,
+	PositiveImbalanceOf, RewardDestination, SessionInterface, StakingLedger, ValidatorPrefs,
 };
 
 use super::pallet::*;
@@ -84,6 +84,38 @@ impl<T: Config> Pallet<T> {
 		StakingLedger::<T>::paired_account(Stash(stash.clone()))
 	}
 
+	/// Inspects and returns the corruption state of a ledger and bond, if any.
+	///
+	/// Note: all operations in this method access directly the `Bonded` and `Ledger` storage maps
+	/// instead of using the [`StakingLedger`] API since the bond and/or ledger may be corrupted.
+	pub(crate) fn inspect_bond_state(
+		stash: &T::AccountId,
+	) -> Result<LedgerIntegrityState, Error<T>> {
+		let lock = T::Currency::balance_locked(crate::STAKING_ID, &stash);
+
+		let controller = <Bonded<T>>::get(stash).ok_or_else(|| {
+			if lock == Zero::zero() {
+				Error::<T>::NotStash
+			} else {
+				Error::<T>::BadState
+			}
+		})?;
+
+		match Ledger::<T>::get(controller) {
+			Some(ledger) =>
+				if ledger.stash != *stash {
+					Ok(LedgerIntegrityState::Corrupted)
+				} else {
+					if lock != ledger.total {
+						Ok(LedgerIntegrityState::LockCorrupted)
+					} else {
+						Ok(LedgerIntegrityState::Ok)
+					}
+				},
+			None => Ok(LedgerIntegrityState::CorruptedKilled),
+		}
+	}
+
 	/// The total balance that can be slashed from a stash account as of right now.
 	pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf<T> {
 		// Weight note: consider making the stake accessible through stash.
@@ -1837,12 +1869,12 @@ impl<T: Config> Pallet<T> {
 			"VoterList contains non-staker"
 		);
 
+		Self::check_ledgers()?;
 		Self::check_bonded_consistency()?;
 		Self::check_payees()?;
 		Self::check_nominators()?;
 		Self::check_exposures()?;
 		Self::check_paged_exposures()?;
-		Self::check_ledgers()?;
 		Self::check_count()
 	}
 
@@ -1851,6 +1883,7 @@ impl<T: Config> Pallet<T> {
 	/// * A bonded (stash, controller) pair should have only one associated ledger. I.e. if the
 	///   ledger is bonded by stash, the controller account must not bond a different ledger.
 	/// * A bonded (stash, controller) pair must have an associated ledger.
+	///
 	/// NOTE: these checks result in warnings only. Once
 	/// <https://github.com/paritytech/polkadot-sdk/issues/3245> is resolved, turn warns into check
 	/// failures.
@@ -1945,19 +1978,18 @@ impl<T: Config> Pallet<T> {
 	}
 
 	/// Invariants:
-	/// * `ledger.controller` is not stored in the storage (but populated at retrieval).
 	/// * Stake consistency: ledger.total == ledger.active + sum(ledger.unlocking).
-	/// * The controller keying the ledger and the ledger stash matches the state of the `Bonded`
-	/// storage.
+	/// * The ledger's controller and stash matches the associated `Bonded` tuple.
+	/// * Staking locked funds for every bonded stash should be the same as its ledger's total.
+	/// * Staking ledger and bond are not corrupted.
 	fn check_ledgers() -> Result<(), TryRuntimeError> {
 		Bonded::<T>::iter()
 			.map(|(stash, ctrl)| {
-				// `ledger.controller` is never stored in raw storage.
-				let raw = Ledger::<T>::get(stash).unwrap_or_else(|| {
-					Ledger::<T>::get(ctrl.clone())
-						.expect("try_check: bonded stash/ctrl does not have an associated ledger")
-				});
-				ensure!(raw.controller.is_none(), "raw storage controller should be None");
+				// ensure locks consistency.
+				ensure!(
+					Self::inspect_bond_state(&stash) == Ok(LedgerIntegrityState::Ok),
+					"bond, ledger and/or staking lock inconsistent for a bonded stash."
+				);
 
 				// ensure ledger consistency.
 				Self::ensure_ledger_consistent(ctrl)

substrate/frame/staking/src/pallet/mod.rs

@@ -25,7 +25,7 @@ use frame_support::{
 	pallet_prelude::*,
 	traits::{
 		Currency, Defensive, DefensiveSaturating, EnsureOrigin, EstimateNextNewSession, Get,
-		LockableCurrency, OnUnbalanced, UnixTime,
+		InspectLockableCurrency, LockableCurrency, OnUnbalanced, UnixTime, WithdrawReasons,
 	},
 	weights::Weight,
 	BoundedVec,
@@ -48,9 +48,9 @@ pub use impls::*;
 
 use crate::{
 	slashing, weights::WeightInfo, AccountIdLookupOf, ActiveEraInfo, BalanceOf, EraPayout,
-	EraRewardPoints, Exposure, ExposurePage, Forcing, MaxNominationsOf, NegativeImbalanceOf,
-	Nominations, NominationsQuota, PositiveImbalanceOf, RewardDestination, SessionInterface,
-	StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs,
+	EraRewardPoints, Exposure, ExposurePage, Forcing, LedgerIntegrityState, MaxNominationsOf,
+	NegativeImbalanceOf, Nominations, NominationsQuota, PositiveImbalanceOf, RewardDestination,
+	SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs,
 };
 
 // The speculative number of spans are used as an input of the weight annotation of
@@ -88,10 +88,10 @@ pub mod pallet {
 	pub trait Config: frame_system::Config {
 		/// The staking balance.
 		type Currency: LockableCurrency<
-			Self::AccountId,
-			Moment = BlockNumberFor<Self>,
-			Balance = Self::CurrencyBalance,
-		>;
+				Self::AccountId,
+				Moment = BlockNumberFor<Self>,
+				Balance = Self::CurrencyBalance,
+			> + InspectLockableCurrency<Self::AccountId>;
 		/// Just the `Currency::Balance` type; we have this item to allow us to constrain it to
 		/// `From<u64>`.
 		type CurrencyBalance: sp_runtime::traits::AtLeast32BitUnsigned
@@ -796,6 +796,7 @@ pub mod pallet {
 	}
 
 	#[pallet::error]
+	#[derive(PartialEq)]
 	pub enum Error<T> {
 		/// Not a controller account.
 		NotController,
@@ -855,6 +856,8 @@ pub mod pallet {
 		BoundNotMet,
 		/// Used when attempting to use deprecated controller account logic.
 		ControllerDeprecated,
+		/// Cannot reset a ledger.
+		CannotRestoreLedger,
 	}
 
 	#[pallet::hooks]
@@ -1980,6 +1983,108 @@ pub mod pallet {
 
 			Ok(Some(T::WeightInfo::deprecate_controller_batch(controllers.len() as u32)).into())
 		}
+
+		/// Restores the state of a ledger which is in an inconsistent state.
+		///
+		/// The requirements to restore a ledger are the following:
+		/// * The stash is bonded; or
+		/// * The stash is not bonded but it has a staking lock left behind; or
+		/// * If the stash has an associated ledger and its state is inconsistent; or
+		/// * If the ledger is not corrupted *but* its staking lock is out of sync.
+		///
+		/// The `maybe_*` input parameters will overwrite the corresponding data and metadata of the
+		/// ledger associated with the stash. If the input parameters are not set, the ledger will
+		/// be reset values from on-chain state.
+		#[pallet::call_index(29)]
+		#[pallet::weight(T::WeightInfo::restore_ledger())]
+		pub fn restore_ledger(
+			origin: OriginFor<T>,
+			stash: T::AccountId,
+			maybe_controller: Option<T::AccountId>,
+			maybe_total: Option<BalanceOf<T>>,
+			maybe_unlocking: Option<BoundedVec<UnlockChunk<BalanceOf<T>>, T::MaxUnlockingChunks>>,
+		) -> DispatchResult {
+			T::AdminOrigin::ensure_origin(origin)?;
+
+			let current_lock = T::Currency::balance_locked(crate::STAKING_ID, &stash);
+			let stash_balance = T::Currency::free_balance(&stash);
+
+			let (new_controller, new_total) = match Self::inspect_bond_state(&stash) {
+				Ok(LedgerIntegrityState::Corrupted) => {
+					let new_controller = maybe_controller.unwrap_or(stash.clone());
+
+					let new_total = if let Some(total) = maybe_total {
+						let new_total = total.min(stash_balance);
+						// enforce lock == ledger.amount.
+						T::Currency::set_lock(
+							crate::STAKING_ID,
+							&stash,
+							new_total,
+							WithdrawReasons::all(),
+						);
+						new_total
+					} else {
+						current_lock
+					};
+
+					Ok((new_controller, new_total))
+				},
+				Ok(LedgerIntegrityState::CorruptedKilled) => {
+					if current_lock == Zero::zero() {
+						// this case needs to restore both lock and ledger, so the new total needs
+						// to be given by the called since there's no way to restore the total
+						// on-chain.
+						ensure!(maybe_total.is_some(), Error::<T>::CannotRestoreLedger);
+						Ok((
+							stash.clone(),
+							maybe_total.expect("total exists as per the check above; qed."),
+						))
+					} else {
+						Ok((stash.clone(), current_lock))
+					}
+				},
+				Ok(LedgerIntegrityState::LockCorrupted) => {
+					// ledger is not corrupted but its locks are out of sync. In this case, we need
+					// to enforce a new ledger.total and staking lock for this stash.
+					let new_total =
+						maybe_total.ok_or(Error::<T>::CannotRestoreLedger)?.min(stash_balance);
+					T::Currency::set_lock(
+						crate::STAKING_ID,
+						&stash,
+						new_total,
+						WithdrawReasons::all(),
+					);
+
+					Ok((stash.clone(), new_total))
+				},
+				Err(Error::<T>::BadState) => {
+					// the stash and ledger do not exist but lock is lingering.
+					T::Currency::remove_lock(crate::STAKING_ID, &stash);
+					ensure!(
+						Self::inspect_bond_state(&stash) == Err(Error::<T>::NotStash),
+						Error::<T>::BadState
+					);
+
+					return Ok(());
+				},
+				Ok(LedgerIntegrityState::Ok) | Err(_) => Err(Error::<T>::CannotRestoreLedger),
+			}?;
+
+			// re-bond stash and controller tuple.
+			Bonded::<T>::insert(&stash, &new_controller);
+
+			// resoter ledger state.
+			let mut ledger = StakingLedger::<T>::new(stash.clone(), new_total);
+			ledger.controller = Some(new_controller);
+			ledger.unlocking = maybe_unlocking.unwrap_or_default();
+			ledger.update()?;
+
+			ensure!(
+				Self::inspect_bond_state(&stash) == Ok(LedgerIntegrityState::Ok),
+				Error::<T>::BadState
+			);
+			Ok(())
+		}
 	}
 }
 

substrate/frame/staking/src/tests.rs

@@ -6933,40 +6933,43 @@ mod ledger {
 			setup_double_bonded_ledgers();
 
 			// Case 1: double bonded but not corrupted:
-			// stash 2 has controller 3:
-			assert_eq!(Bonded::<Test>::get(2), Some(3));
-			assert_eq!(Ledger::<Test>::get(3).unwrap().stash, 2);
+			// stash 444 has controller 555:
+			assert_eq!(Bonded::<Test>::get(444), Some(555));
+			assert_eq!(Ledger::<Test>::get(555).unwrap().stash, 444);
 
-			// stash 2 is also a controller of 1:
-			assert_eq!(Bonded::<Test>::get(1), Some(2));
-			assert_eq!(StakingLedger::<Test>::paired_account(StakingAccount::Stash(1)), Some(2));
-			assert_eq!(Ledger::<Test>::get(2).unwrap().stash, 1);
+			// stash 444 is also a controller of 333:
+			assert_eq!(Bonded::<Test>::get(333), Some(444));
+			assert_eq!(
+				StakingLedger::<Test>::paired_account(StakingAccount::Stash(333)),
+				Some(444)
+			);
+			assert_eq!(Ledger::<Test>::get(444).unwrap().stash, 333);
 
-			// although 2 is double bonded (it is a controller and a stash of different ledgers),
+			// although 444 is double bonded (it is a controller and a stash of different ledgers),
 			// we can safely retrieve the ledger and mutate it since the correct ledger is
 			// returned.
-			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Stash(2));
-			assert_eq!(ledger_result.unwrap().stash, 2); // correct ledger.
+			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Stash(444));
+			assert_eq!(ledger_result.unwrap().stash, 444); // correct ledger.
 
-			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Controller(2));
-			assert_eq!(ledger_result.unwrap().stash, 1); // correct ledger.
+			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Controller(444));
+			assert_eq!(ledger_result.unwrap().stash, 333); // correct ledger.
 
-			// fetching ledger 1 by its stash works.
-			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Stash(1));
-			assert_eq!(ledger_result.unwrap().stash, 1);
+			// fetching ledger 333 by its stash works.
+			let ledger_result = StakingLedger::<Test>::get(StakingAccount::Stash(333));
+			assert_eq!(ledger_result.unwrap().stash, 333);
 
 			// Case 2: corrupted ledger bonding.
 			// in this case, we simulate what happens when fetching a ledger by stash returns a
 			// ledger with a different stash. when this happens, we return an error instead of the
 			// ledger to prevent ledger mutations.
-			let mut ledger = Ledger::<Test>::get(2).unwrap();
-			assert_eq!(ledger.stash, 1);
-			ledger.stash = 2;
-			Ledger::<Test>::insert(2, ledger);
+			let mut ledger = Ledger::<Test>::get(444).unwrap();
+			assert_eq!(ledger.stash, 333);
+			ledger.stash = 444;
+			Ledger::<Test>::insert(444, ledger);
 
 			// now, we are prevented from fetching the ledger by stash from 1. It's associated
 			// controller (2) is now bonding a ledger with a different stash (2, not 1).
-			assert!(StakingLedger::<Test>::get(StakingAccount::Stash(1)).is_err());
+			assert!(StakingLedger::<Test>::get(StakingAccount::Stash(333)).is_err());
 		})
 	}
 
@@ -7069,7 +7072,7 @@ mod ledger {
 
 	#[test]
 	fn deprecate_controller_batch_works_full_weight() {
-		ExtBuilder::default().build_and_execute(|| {
+		ExtBuilder::default().try_state(false).build_and_execute(|| {
 			// Given:
 
 			let start = 1001;
@@ -7253,7 +7256,7 @@ mod ledger {
 			let bounded_controllers: BoundedVec<
 				_,
 				<Test as Config>::MaxControllersInDeprecationBatch,
-			> = BoundedVec::try_from(vec![1, 2, 3, 4]).unwrap();
+			> = BoundedVec::try_from(vec![333, 444, 555, 777]).unwrap();
 
 			assert_ok!(Staking::deprecate_controller_batch(
 				RuntimeOrigin::root(),
@@ -7276,7 +7279,7 @@ mod ledger {
 			let bounded_controllers: BoundedVec<
 				_,
 				<Test as Config>::MaxControllersInDeprecationBatch,
-			> = BoundedVec::try_from(vec![4, 3, 2, 1]).unwrap();
+			> = BoundedVec::try_from(vec![777, 555, 444, 333]).unwrap();
 
 			assert_ok!(Staking::deprecate_controller_batch(
 				RuntimeOrigin::root(),
@@ -7296,9 +7299,9 @@ mod ledger {
 			setup_double_bonded_ledgers();
 
 			// in this case, setting controller works due to the ordering of the calls.
-			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(1)));
-			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(2)));
-			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(3)));
+			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(333)));
+			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(444)));
+			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(555)));
 		})
 	}
 
@@ -7307,17 +7310,400 @@ mod ledger {
 		ExtBuilder::default().has_stakers(false).try_state(false).build_and_execute(|| {
 			setup_double_bonded_ledgers();
 
-			// setting the controller of ledger associated with stash 3 fails since its stash is a
+			// setting the controller of ledger associated with stash 555 fails since its stash is a
 			// controller of another ledger.
 			assert_noop!(
-				Staking::set_controller(RuntimeOrigin::signed(3)),
+				Staking::set_controller(RuntimeOrigin::signed(555)),
 				Error::<Test>::BadState
 			);
 			assert_noop!(
-				Staking::set_controller(RuntimeOrigin::signed(2)),
+				Staking::set_controller(RuntimeOrigin::signed(444)),
 				Error::<Test>::BadState
 			);
-			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(1)));
+			assert_ok!(Staking::set_controller(RuntimeOrigin::signed(333)));
+		})
+	}
+}
+
+mod ledger_recovery {
+	use super::*;
+	use frame_support::traits::InspectLockableCurrency;
+
+	#[test]
+	fn inspect_recovery_ledger_simple_works() {
+		ExtBuilder::default().has_stakers(true).try_state(false).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			// non corrupted ledger.
+			assert_eq!(Staking::inspect_bond_state(&11).unwrap(), LedgerIntegrityState::Ok);
+
+			// non bonded stash.
+			assert!(Bonded::<Test>::get(&1111).is_none());
+			assert!(Staking::inspect_bond_state(&1111).is_err());
+
+			// double bonded but not corrupted.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+		})
+	}
+
+	#[test]
+	fn inspect_recovery_ledger_corupted_killed_works() {
+		ExtBuilder::default().has_stakers(true).try_state(false).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			let lock_333_before = Balances::balance_locked(crate::STAKING_ID, &333);
+
+			// get into corrupted and killed ledger state by killing a corrupted ledger:
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			// kill(333)
+			// (444, 444) -> corrupted and None.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// 333 is corrupted since it's controller is linking 444 ledger.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Corrupted);
+			// 444 however is OK.
+			assert_eq!(Staking::inspect_bond_state(&444).unwrap(), LedgerIntegrityState::Ok);
+
+			// kill the corrupted ledger that is associated with stash 333.
+			assert_ok!(StakingLedger::<Test>::kill(&333));
+
+			// 333 bond is no more but it returns `BadState` because the lock on this stash is
+			// still set (see checks below).
+			assert_eq!(Staking::inspect_bond_state(&333), Err(Error::<Test>::BadState));
+			// now the *other* ledger associated with 444 has been corrupted and killed (None).
+			assert_eq!(
+				Staking::inspect_bond_state(&444),
+				Ok(LedgerIntegrityState::CorruptedKilled)
+			);
+
+			// side effects on 333 - ledger, bonded, payee, lock should be completely empty.
+			// however, 333 lock remains.
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &333), lock_333_before); // NOK
+			assert!(Bonded::<Test>::get(&333).is_none()); // OK
+			assert!(Payee::<Test>::get(&333).is_none()); // OK
+			assert!(Ledger::<Test>::get(&444).is_none()); // OK
+
+			// side effects on 444 - ledger, bonded, payee, lock should remain be intact.
+			// however, 444 lock was removed.
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &444), 0); // NOK
+			assert!(Bonded::<Test>::get(&444).is_some()); // OK
+			assert!(Payee::<Test>::get(&444).is_some()); // OK
+			assert!(Ledger::<Test>::get(&555).is_none()); // NOK
+
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+		})
+	}
+
+	#[test]
+	fn inspect_recovery_ledger_corupted_killed_other_works() {
+		ExtBuilder::default().has_stakers(true).try_state(false).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			let lock_333_before = Balances::balance_locked(crate::STAKING_ID, &333);
+
+			// get into corrupted and killed ledger state by killing a corrupted ledger:
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			// kill(444)
+			// (333, 444) -> corrupted and None
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// 333 is corrupted since it's controller is linking 444 ledger.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Corrupted);
+			// 444 however is OK.
+			assert_eq!(Staking::inspect_bond_state(&444).unwrap(), LedgerIntegrityState::Ok);
+
+			// kill the *other* ledger that is double bonded but not corrupted.
+			assert_ok!(StakingLedger::<Test>::kill(&444));
+
+			// now 333 is corrupted and None through the *other* ledger being killed.
+			assert_eq!(
+				Staking::inspect_bond_state(&333).unwrap(),
+				LedgerIntegrityState::CorruptedKilled,
+			);
+			// 444 is cleaned and not a stash anymore; no lock left behind.
+			assert_eq!(Ledger::<Test>::get(&444), None);
+			assert_eq!(Staking::inspect_bond_state(&444), Err(Error::<Test>::NotStash));
+
+			// side effects on 333 - ledger, bonded, payee, lock should be intact.
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &333), lock_333_before); // OK
+			assert_eq!(Bonded::<Test>::get(&333), Some(444)); // OK
+			assert!(Payee::<Test>::get(&333).is_some()); // OK
+											 // however, ledger associated with its controller was killed.
+			assert!(Ledger::<Test>::get(&444).is_none()); // NOK
+
+			// side effects on 444 - ledger, bonded, payee, lock should be completely removed.
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &444), 0); // OK
+			assert!(Bonded::<Test>::get(&444).is_none()); // OK
+			assert!(Payee::<Test>::get(&444).is_none()); // OK
+			assert!(Ledger::<Test>::get(&555).is_none()); // OK
+
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+		})
+	}
+
+	#[test]
+	fn inspect_recovery_ledger_lock_corrupted_works() {
+		ExtBuilder::default().has_stakers(true).try_state(false).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			// get into lock corrupted ledger state by bond_extra on a ledger that is double bonded
+			// with a corrupted ledger.
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			//  bond_extra(333, 10) -> lock corrupted on 444
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+			bond_extra_no_checks(&333, 10);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// 333 is corrupted since it's controller is linking 444 ledger.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Corrupted);
+			// 444 ledger is not corrupted but locks got out of sync.
+			assert_eq!(
+				Staking::inspect_bond_state(&444).unwrap(),
+				LedgerIntegrityState::LockCorrupted
+			);
+		})
+	}
+
+	// Corrupted ledger restore.
+	//
+	// * Double bonded and corrupted ledger.
+	#[test]
+	fn restore_ledger_corrupted_works() {
+		ExtBuilder::default().has_stakers(true).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			// get into corrupted and killed ledger state.
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Corrupted);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// recover the ledger bonded by 333 stash.
+			assert_ok!(Staking::restore_ledger(RuntimeOrigin::root(), 333, None, None, None));
+
+			// try-state checks are ok now.
+			assert_ok!(Staking::do_try_state(System::block_number()));
+		})
+	}
+
+	// Corrupted and killed ledger restore.
+	//
+	// * Double bonded and corrupted ledger.
+	// * Ledger killed by own controller.
+	#[test]
+	fn restore_ledger_corrupted_killed_works() {
+		ExtBuilder::default().has_stakers(true).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			// ledger.total == lock
+			let total_444_before_corruption = Balances::balance_locked(crate::STAKING_ID, &444);
+
+			// get into corrupted and killed ledger state by killing a corrupted ledger:
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			// kill(333)
+			// (444, 444) -> corrupted and None.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			// kill the corrupted ledger that is associated with stash 333.
+			assert_ok!(StakingLedger::<Test>::kill(&333));
+
+			// 333 bond is no more but it returns `BadState` because the lock on this stash is
+			// still set (see checks below).
+			assert_eq!(Staking::inspect_bond_state(&333), Err(Error::<Test>::BadState));
+			// now the *other* ledger associated with 444 has been corrupted and killed (None).
+			assert!(Staking::ledger(StakingAccount::Stash(444)).is_err());
+
+			// try-state should fail.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// recover the ledger bonded by 333 stash.
+			assert_ok!(Staking::restore_ledger(RuntimeOrigin::root(), 333, None, None, None));
+
+			// for the try-state checks to pass, we also need to recover the stash 444 which is
+			// corrupted too by proxy of kill(333). Currently, both the lock and the ledger of 444
+			// have been cleared so we need to provide the new amount to restore the ledger.
+			assert_noop!(
+				Staking::restore_ledger(RuntimeOrigin::root(), 444, None, None, None),
+				Error::<Test>::CannotRestoreLedger
+			);
+
+			assert_ok!(Staking::restore_ledger(
+				RuntimeOrigin::root(),
+				444,
+				None,
+				Some(total_444_before_corruption),
+				None,
+			));
+
+			// try-state checks are ok now.
+			assert_ok!(Staking::do_try_state(System::block_number()));
+		})
+	}
+
+	// Corrupted and killed by *other* ledger restore.
+	//
+	// * Double bonded and corrupted ledger.
+	// * Ledger killed by own controller.
+	#[test]
+	fn restore_ledger_corrupted_killed_other_works() {
+		ExtBuilder::default().has_stakers(true).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			// get into corrupted and killed ledger state by killing a corrupted ledger:
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			// kill(444)
+			// (333, 444) -> corrupted and None
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// 333 is corrupted since it's controller is linking 444 ledger.
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Corrupted);
+			// 444 however is OK.
+			assert_eq!(Staking::inspect_bond_state(&444).unwrap(), LedgerIntegrityState::Ok);
+
+			// kill the *other* ledger that is double bonded but not corrupted.
+			assert_ok!(StakingLedger::<Test>::kill(&444));
+
+			// recover the ledger bonded by 333 stash.
+			assert_ok!(Staking::restore_ledger(RuntimeOrigin::root(), 333, None, None, None));
+
+			// 444 does not need recover in this case since it's been killed successfully.
+			assert_eq!(Staking::inspect_bond_state(&444), Err(Error::<Test>::NotStash));
+
+			// try-state checks are ok now.
+			assert_ok!(Staking::do_try_state(System::block_number()));
+		})
+	}
+
+	// Corrupted with bond_extra.
+	//
+	// * Double bonded and corrupted ledger.
+	// * Corrupted ledger calls `bond_extra`
+	#[test]
+	fn restore_ledger_corrupted_bond_extra_works() {
+		ExtBuilder::default().has_stakers(true).build_and_execute(|| {
+			setup_double_bonded_ledgers();
+
+			let lock_333_before = Balances::balance_locked(crate::STAKING_ID, &333);
+			let lock_444_before = Balances::balance_locked(crate::STAKING_ID, &444);
+
+			// get into corrupted and killed ledger state by killing a corrupted ledger:
+			// init state:
+			//  (333, 444)
+			//  (444, 555)
+			// set_controller(444) to 444
+			//  (333, 444) -> corrupted
+			//  (444, 444)
+			// bond_extra(444, 40) -> OK
+			// bond_extra(333, 30) -> locks out of sync
+
+			assert_eq!(Staking::inspect_bond_state(&333).unwrap(), LedgerIntegrityState::Ok);
+			set_controller_no_checks(&444);
+
+			// now try-state fails.
+			assert!(Staking::do_try_state(System::block_number()).is_err());
+
+			// if 444 bonds extra, the locks remain in sync.
+			bond_extra_no_checks(&444, 40);
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &333), lock_333_before);
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &444), lock_444_before + 40);
+
+			// however if 333 bonds extra, the wrong lock is updated.
+			bond_extra_no_checks(&333, 30);
+			assert_eq!(
+				Balances::balance_locked(crate::STAKING_ID, &333),
+				lock_444_before + 40 + 30
+			); //not OK
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &444), lock_444_before + 40); // OK
+
+			// recover the ledger bonded by 333 stash. Note that the total/lock needs to be
+			// re-written since on-chain data lock has become out of sync.
+			assert_ok!(Staking::restore_ledger(
+				RuntimeOrigin::root(),
+				333,
+				None,
+				Some(lock_333_before + 30),
+				None
+			));
+
+			// now recover 444 that although it's not corrupted, its lock and ledger.total are out
+			// of sync. in which case, we need to explicitly set the ledger's lock and amount,
+			// otherwise the ledger recover will fail.
+			assert_noop!(
+				Staking::restore_ledger(RuntimeOrigin::root(), 444, None, None, None),
+				Error::<Test>::CannotRestoreLedger
+			);
+
+			//and enforcing a new ledger lock/total on this non-corrupted ledger will work.
+			assert_ok!(Staking::restore_ledger(
+				RuntimeOrigin::root(),
+				444,
+				None,
+				Some(lock_444_before + 40),
+				None
+			));
+
+			// double-check that ledgers got to expected state and bond_extra done during the
+			// corrupted state is part of the recovered ledgers.
+			let ledger_333 = Bonded::<Test>::get(&333).and_then(Ledger::<Test>::get).unwrap();
+			let ledger_444 = Bonded::<Test>::get(&444).and_then(Ledger::<Test>::get).unwrap();
+
+			assert_eq!(ledger_333.total, lock_333_before + 30);
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &333), ledger_333.total);
+			assert_eq!(ledger_444.total, lock_444_before + 40);
+			assert_eq!(Balances::balance_locked(crate::STAKING_ID, &444), ledger_444.total);
+
+			// try-state checks are ok now.
+			assert_ok!(Staking::do_try_state(System::block_number()));
 		})
 	}
 }