migration(democracy): unreserve deposits and clear locks (#14226)

* unlock and unreserve all migration

* log total amount to unlock

* clippy

* clippy

* simplify balance types

* add tests

* fix comment

* fix std features

* remove redundant dev-dep

* address comments

* remove redundant std

* estimate weights

* fix comments

* fix tests

* clippy

* track democracy bugged deposits

* improve pre_migration logs

* oliver comments from tips pr

* run pre and post hooks in tests

* flexible dbweight type

* Update frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* docs

* Update frame/democracy/src/lib.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update frame/democracy/src/migrations/unlock_and_unreserve_all_funds.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* add log target

* fix visibility

* lint

* use log target

---------

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
This commit is contained in:
Liam Aharon
2023-06-14 02:06:35 +10:00
committed by GitHub
parent 3b6d31f03d
commit 93dea08420
4 changed files with 375 additions and 2 deletions
@@ -0,0 +1,22 @@
// This file is part of Substrate.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! All migrations of this pallet.
/// Migration to unlock and unreserve all pallet funds.
pub mod unlock_and_unreserve_all_funds;
/// V1 storage migrations for the preimage pallet.
pub mod v1;
@@ -0,0 +1,351 @@
// This file is part of Substrate.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! A migration that unreserves all deposit and unlocks all stake held in the context of this
//! pallet.
use crate::{BalanceOf, DEMOCRACY_ID};
use core::iter::Sum;
use frame_support::traits::{LockableCurrency, OnRuntimeUpgrade, ReservableCurrency};
use sp_core::Get;
use sp_runtime::{traits::Zero, Saturating};
use sp_std::{collections::btree_map::BTreeMap, vec::Vec};
const LOG_TARGET: &str = "runtime::democracy::migrations::unlock_and_unreserve_all_funds";
/// A migration that unreserves all deposit and unlocks all stake held in the context of this
/// pallet.
///
/// Useful to prevent funds from being locked up when the pallet is being deprecated.
///
/// The pallet should be made inoperable before this migration is run.
///
/// (See also [`RemovePallet`][frame_support::migrations::RemovePallet])
pub struct UnlockAndUnreserveAllFunds<T: crate::Config>(sp_std::marker::PhantomData<T>);
impl<T: crate::Config> UnlockAndUnreserveAllFunds<T> {
/// Calculates and returns the total amounts reserved by each account by this pallet, and all
/// accounts with locks in the context of this pallet.
///
/// There is no need to return the amount locked, because the entire lock is removed (always
/// should be zero post-migration). We need to return the amounts reserved to check that the
/// reserved amount is deducted correctly.
///
/// # Returns
///
/// This function returns a tuple of two `BTreeMap` collections and the weight of the reads:
///
/// * `BTreeMap<T::AccountId, BalanceOf<T>>`: Map of account IDs to their respective total
/// reserved balance by this pallet
/// * `BTreeMap<T::AccountId, BalanceOf<T>>`: Map of account IDs to their respective total
/// locked balance by this pallet
/// * `frame_support::weights::Weight`: the weight consumed by this call.
fn get_account_deposits_and_locks() -> (
BTreeMap<T::AccountId, BalanceOf<T>>,
BTreeMap<T::AccountId, BalanceOf<T>>,
frame_support::weights::Weight,
) {
let mut deposit_of_len = 0;
// Get all deposits (reserved).
let mut total_voting_vec_entries: u64 = 0;
let account_deposits: BTreeMap<T::AccountId, BalanceOf<T>> = crate::DepositOf::<T>::iter()
.flat_map(|(_prop_index, (accounts, balance))| {
// Count the number of deposits
deposit_of_len.saturating_inc();
// Track the total number of vec entries to calculate the weight of the reads.
total_voting_vec_entries.saturating_accrue(accounts.len() as u64);
// Create a vec of tuples where each account is associated with the given balance
accounts.into_iter().map(|account| (account, balance)).collect::<Vec<_>>()
})
.fold(BTreeMap::new(), |mut acc, (account, balance)| {
// Add the balance to the account's existing balance in the accumulator
acc.entry(account.clone()).or_insert(Zero::zero()).saturating_accrue(balance);
acc
});
// Voter accounts have amounts locked.
let account_stakes: BTreeMap<T::AccountId, BalanceOf<T>> = crate::VotingOf::<T>::iter()
.map(|(account_id, voting)| (account_id, voting.locked_balance()))
.collect();
let voting_of_len = account_stakes.len() as u64;
(
account_deposits,
account_stakes,
T::DbWeight::get().reads(
deposit_of_len.saturating_add(voting_of_len).saturating_add(
// Max items in a Voting enum is MaxVotes + 5
total_voting_vec_entries
.saturating_mul(T::MaxVotes::get().saturating_add(5) as u64),
),
),
)
}
}
impl<T: crate::Config> OnRuntimeUpgrade for UnlockAndUnreserveAllFunds<T>
where
BalanceOf<T>: Sum,
{
/// Collects pre-migration data useful for validating the migration was successful, and also
/// checks the integrity of deposited and reserved balances.
///
/// Steps:
/// 1. Gets the deposited balances for each account stored in this pallet.
/// 2. Collects actual pre-migration reserved balances for each account.
/// 3. Checks the integrity of the deposited balances.
/// 4. Prints summary statistics about the state to be migrated.
/// 5. Encodes and returns pre-migration data to be used in post_upgrade.
///
/// Fails with a `TryRuntimeError` if somehow the amount reserved by this pallet is greater than
/// the actual total reserved amount for any accounts.
#[cfg(feature = "try-runtime")]
fn pre_upgrade() -> Result<Vec<u8>, sp_runtime::TryRuntimeError> {
use codec::Encode;
use sp_std::collections::btree_set::BTreeSet;
// Get staked and deposited balances as reported by this pallet.
let (account_deposits, account_locks, _) = Self::get_account_deposits_and_locks();
let all_accounts = account_deposits
.keys()
.chain(account_locks.keys())
.cloned()
.collect::<BTreeSet<_>>();
let account_reserved_before: BTreeMap<T::AccountId, BalanceOf<T>> = account_deposits
.keys()
.map(|account| (account.clone(), T::Currency::reserved_balance(&account)))
.collect();
// Total deposited for each account *should* be less than or equal to the total reserved,
// however this does not hold for all cases due to bugs in the reserve logic of this pallet.
let bugged_deposits = all_accounts
.iter()
.filter(|account| {
account_deposits.get(&account).unwrap_or(&Zero::zero()) >
account_reserved_before.get(&account).unwrap_or(&Zero::zero())
})
.count();
let total_deposits_to_unreserve =
account_deposits.clone().into_values().sum::<BalanceOf<T>>();
let total_stake_to_unlock = account_locks.clone().into_values().sum::<BalanceOf<T>>();
log::info!(target: LOG_TARGET, "Total accounts: {:?}", all_accounts.len());
log::info!(target: LOG_TARGET, "Total stake to unlock: {:?}", total_stake_to_unlock);
log::info!(target: LOG_TARGET, "Total deposit to unreserve: {:?}", total_deposits_to_unreserve);
log::info!(target: LOG_TARGET, "Bugged deposits: {}/{}", bugged_deposits, account_deposits.len());
Ok(account_reserved_before.encode())
}
/// Executes the migration.
///
/// Steps:
/// 1. Retrieves the deposit and accounts with locks for the pallet.
/// 2. Unreserves the deposited funds for each account.
/// 3. Unlocks the staked funds for each account.
fn on_runtime_upgrade() -> frame_support::weights::Weight {
// Get staked and deposited balances as reported by this pallet.
let (account_deposits, account_stakes, initial_reads) =
Self::get_account_deposits_and_locks();
// Deposited funds need to be unreserved.
for (account, unreserve_amount) in account_deposits.iter() {
if unreserve_amount.is_zero() {
log::warn!(target: LOG_TARGET, "Unexpected zero amount to unreserve!");
continue
}
T::Currency::unreserve(&account, *unreserve_amount);
}
// Staked funds need to be unlocked.
for account in account_stakes.keys() {
T::Currency::remove_lock(DEMOCRACY_ID, account);
}
T::DbWeight::get()
.reads_writes(
account_stakes.len().saturating_add(account_deposits.len()) as u64,
account_stakes.len().saturating_add(account_deposits.len()) as u64,
)
.saturating_add(initial_reads)
}
/// Performs post-upgrade sanity checks:
///
/// 1. No locks remain for this pallet in Balances.
/// 2. The reserved balance for each account has been reduced by the expected amount.
#[cfg(feature = "try-runtime")]
fn post_upgrade(
account_reserved_before_bytes: Vec<u8>,
) -> Result<(), sp_runtime::TryRuntimeError> {
use codec::Decode;
let account_reserved_before =
BTreeMap::<T::AccountId, BalanceOf<T>>::decode(&mut &account_reserved_before_bytes[..])
.map_err(|_| "Failed to decode account_reserved_before_bytes")?;
// Get staked and deposited balances as reported by this pallet.
let (account_deposits, _, _) = Self::get_account_deposits_and_locks();
// Check that the reserved balance is reduced by the expected deposited amount.
for (account, actual_reserved_before) in account_reserved_before {
let actual_reserved_after = T::Currency::reserved_balance(&account);
let expected_amount_deducted = *account_deposits
.get(&account)
.expect("account deposit must exist to be in pre_migration_data, qed");
let expected_reserved_after =
actual_reserved_before.saturating_sub(expected_amount_deducted);
assert!(
actual_reserved_after == expected_reserved_after,
"Reserved balance for {:?} is incorrect. actual before: {:?}, actual after, {:?}, expected deducted: {:?}",
account,
actual_reserved_before,
actual_reserved_after,
expected_amount_deducted,
);
}
Ok(())
}
}
#[cfg(all(feature = "try-runtime", test))]
mod test {
use super::*;
use crate::{
tests::{new_test_ext, Test},
DepositOf, Voting, VotingOf,
};
use frame_support::{
assert_ok,
traits::{Currency, OnRuntimeUpgrade, ReservableCurrency, WithdrawReasons},
BoundedVec,
};
#[test]
fn unreserve_works_for_depositer() {
let depositer_0 = 10;
let depositer_1 = 11;
let deposit = 25;
let depositer_0_initial_reserved = 0;
let depositer_1_initial_reserved = 15;
let initial_balance = 100_000;
new_test_ext().execute_with(|| {
// Set up initial state.
<Test as crate::Config>::Currency::make_free_balance_be(&depositer_0, initial_balance);
<Test as crate::Config>::Currency::make_free_balance_be(&depositer_1, initial_balance);
assert_ok!(<Test as crate::Config>::Currency::reserve(
&depositer_0,
depositer_0_initial_reserved + deposit
));
assert_ok!(<Test as crate::Config>::Currency::reserve(
&depositer_1,
depositer_1_initial_reserved + deposit
));
let depositors =
BoundedVec::<_, <Test as crate::Config>::MaxDeposits>::truncate_from(vec![
depositer_0,
depositer_1,
]);
DepositOf::<Test>::insert(0, (depositors, deposit));
// Sanity check: ensure initial reserved balance was set correctly.
assert_eq!(
<Test as crate::Config>::Currency::reserved_balance(&depositer_0),
depositer_0_initial_reserved + deposit
);
assert_eq!(
<Test as crate::Config>::Currency::reserved_balance(&depositer_1),
depositer_1_initial_reserved + deposit
);
// Run the migration.
let bytes = UnlockAndUnreserveAllFunds::<Test>::pre_upgrade()
.unwrap_or_else(|e| panic!("pre_upgrade failed: {:?}", e));
UnlockAndUnreserveAllFunds::<Test>::on_runtime_upgrade();
assert_ok!(UnlockAndUnreserveAllFunds::<Test>::post_upgrade(bytes));
// Assert the reserved balance was reduced by the expected amount.
assert_eq!(
<Test as crate::Config>::Currency::reserved_balance(&depositer_0),
depositer_0_initial_reserved
);
assert_eq!(
<Test as crate::Config>::Currency::reserved_balance(&depositer_1),
depositer_1_initial_reserved
);
});
}
#[test]
fn unlock_works_for_voter() {
let voter = 10;
let stake = 25;
let initial_locks = vec![(b"somethin", 10)];
let initial_balance = 100_000;
new_test_ext().execute_with(|| {
// Set up initial state.
<Test as crate::Config>::Currency::make_free_balance_be(&voter, initial_balance);
for lock in initial_locks.clone() {
<Test as crate::Config>::Currency::set_lock(
*lock.0,
&voter,
lock.1,
WithdrawReasons::all(),
);
}
VotingOf::<Test>::insert(voter, Voting::default());
<Test as crate::Config>::Currency::set_lock(
DEMOCRACY_ID,
&voter,
stake,
WithdrawReasons::all(),
);
// Sanity check: ensure initial Balance state was set up correctly.
let mut voter_all_locks = initial_locks.clone();
voter_all_locks.push((&DEMOCRACY_ID, stake));
assert_eq!(
<Test as crate::Config>::Currency::locks(&voter)
.iter()
.map(|lock| (&lock.id, lock.amount))
.collect::<Vec<_>>(),
voter_all_locks
);
// Run the migration.
let bytes = UnlockAndUnreserveAllFunds::<Test>::pre_upgrade()
.unwrap_or_else(|e| panic!("pre_upgrade failed: {:?}", e));
UnlockAndUnreserveAllFunds::<Test>::on_runtime_upgrade();
assert_ok!(UnlockAndUnreserveAllFunds::<Test>::post_upgrade(bytes));
// Assert the voter lock was removed
assert_eq!(
<Test as crate::Config>::Currency::locks(&voter)
.iter()
.map(|lock| (&lock.id, lock.amount))
.collect::<Vec<_>>(),
initial_locks
);
});
}
}
@@ -0,0 +1,236 @@
// This file is part of Substrate.
// Copyright (C) Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Storage migrations for the preimage pallet.
use crate::*;
use frame_support::{pallet_prelude::*, storage_alias, traits::OnRuntimeUpgrade, BoundedVec};
use sp_core::H256;
/// The log target.
const TARGET: &'static str = "runtime::democracy::migration::v1";
/// The original data layout of the democracy pallet without a specific version number.
mod v0 {
use super::*;
#[storage_alias]
pub type PublicProps<T: Config> = StorageValue<
Pallet<T>,
Vec<(PropIndex, <T as frame_system::Config>::Hash, <T as frame_system::Config>::AccountId)>,
ValueQuery,
>;
#[storage_alias]
pub type NextExternal<T: Config> =
StorageValue<Pallet<T>, (<T as frame_system::Config>::Hash, VoteThreshold)>;
#[cfg(feature = "try-runtime")]
#[storage_alias]
pub type ReferendumInfoOf<T: Config> = StorageMap<
Pallet<T>,
frame_support::Twox64Concat,
ReferendumIndex,
ReferendumInfo<
<T as frame_system::Config>::BlockNumber,
<T as frame_system::Config>::Hash,
BalanceOf<T>,
>,
>;
}
pub mod v1 {
use super::*;
/// Migration for translating bare `Hash`es into `Bounded<Call>`s.
pub struct Migration<T>(sp_std::marker::PhantomData<T>);
impl<T: Config + frame_system::Config<Hash = H256>> OnRuntimeUpgrade for Migration<T> {
#[cfg(feature = "try-runtime")]
fn pre_upgrade() -> Result<Vec<u8>, sp_runtime::TryRuntimeError> {
ensure!(StorageVersion::get::<Pallet<T>>() == 0, "can only upgrade from version 0");
let props_count = v0::PublicProps::<T>::get().len();
log::info!(target: TARGET, "{} public proposals will be migrated.", props_count,);
ensure!(props_count <= T::MaxProposals::get() as usize, Error::<T>::TooMany);
let referenda_count = v0::ReferendumInfoOf::<T>::iter().count();
log::info!(target: TARGET, "{} referenda will be migrated.", referenda_count);
Ok((props_count as u32, referenda_count as u32).encode())
}
#[allow(deprecated)]
fn on_runtime_upgrade() -> Weight {
let mut weight = T::DbWeight::get().reads(1);
if StorageVersion::get::<Pallet<T>>() != 0 {
log::warn!(
target: TARGET,
"skipping on_runtime_upgrade: executed on wrong storage version.\
Expected version 0"
);
return weight
}
ReferendumInfoOf::<T>::translate(
|index, old: ReferendumInfo<T::BlockNumber, T::Hash, BalanceOf<T>>| {
weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1));
log::info!(target: TARGET, "migrating referendum #{:?}", &index);
Some(match old {
ReferendumInfo::Ongoing(status) =>
ReferendumInfo::Ongoing(ReferendumStatus {
end: status.end,
proposal: Bounded::from_legacy_hash(status.proposal),
threshold: status.threshold,
delay: status.delay,
tally: status.tally,
}),
ReferendumInfo::Finished { approved, end } =>
ReferendumInfo::Finished { approved, end },
})
},
);
let props = v0::PublicProps::<T>::take()
.into_iter()
.map(|(i, hash, a)| (i, Bounded::from_legacy_hash(hash), a))
.collect::<Vec<_>>();
let bounded = BoundedVec::<_, T::MaxProposals>::truncate_from(props.clone());
PublicProps::<T>::put(bounded);
weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1));
if props.len() as u32 > T::MaxProposals::get() {
log::error!(
target: TARGET,
"truncated {} public proposals to {}; continuing",
props.len(),
T::MaxProposals::get()
);
}
if let Some((hash, threshold)) = v0::NextExternal::<T>::take() {
log::info!(target: TARGET, "migrating next external proposal");
NextExternal::<T>::put((Bounded::from_legacy_hash(hash), threshold));
}
StorageVersion::new(1).put::<Pallet<T>>();
weight.saturating_add(T::DbWeight::get().reads_writes(1, 2))
}
#[cfg(feature = "try-runtime")]
fn post_upgrade(state: Vec<u8>) -> Result<(), sp_runtime::TryRuntimeError> {
ensure!(StorageVersion::get::<Pallet<T>>() == 1, "must upgrade");
let (old_props_count, old_ref_count): (u32, u32) =
Decode::decode(&mut &state[..]).expect("pre_upgrade provides a valid state; qed");
let new_props_count = crate::PublicProps::<T>::get().len() as u32;
ensure!(new_props_count == old_props_count, "must migrate all public proposals");
let new_ref_count = crate::ReferendumInfoOf::<T>::iter().count() as u32;
ensure!(new_ref_count == old_ref_count, "must migrate all referenda");
log::info!(
target: TARGET,
"{} public proposals migrated, {} referenda migrated",
new_props_count,
new_ref_count,
);
Ok(())
}
}
}
#[cfg(test)]
#[cfg(feature = "try-runtime")]
mod test {
use super::*;
use crate::{
tests::{Test as T, *},
types::*,
};
use frame_support::bounded_vec;
#[allow(deprecated)]
#[test]
fn migration_works() {
new_test_ext().execute_with(|| {
assert_eq!(StorageVersion::get::<Pallet<T>>(), 0);
// Insert some values into the v0 storage:
// Case 1: Ongoing referendum
let hash = H256::repeat_byte(1);
let status = ReferendumStatus {
end: 1u32.into(),
proposal: hash.clone(),
threshold: VoteThreshold::SuperMajorityApprove,
delay: 1u32.into(),
tally: Tally { ayes: 1u32.into(), nays: 1u32.into(), turnout: 1u32.into() },
};
v0::ReferendumInfoOf::<T>::insert(1u32, ReferendumInfo::Ongoing(status));
// Case 2: Finished referendum
v0::ReferendumInfoOf::<T>::insert(
2u32,
ReferendumInfo::Finished { approved: true, end: 123u32.into() },
);
// Case 3: Public proposals
let hash2 = H256::repeat_byte(2);
v0::PublicProps::<T>::put(vec![
(3u32, hash.clone(), 123u64),
(4u32, hash2.clone(), 123u64),
]);
// Case 4: Next external
v0::NextExternal::<T>::put((hash.clone(), VoteThreshold::SuperMajorityApprove));
// Migrate.
let state = v1::Migration::<T>::pre_upgrade().unwrap();
let _weight = v1::Migration::<T>::on_runtime_upgrade();
v1::Migration::<T>::post_upgrade(state).unwrap();
// Check that all values got migrated.
// Case 1: Ongoing referendum
assert_eq!(
ReferendumInfoOf::<T>::get(1u32),
Some(ReferendumInfo::Ongoing(ReferendumStatus {
end: 1u32.into(),
proposal: Bounded::from_legacy_hash(hash),
threshold: VoteThreshold::SuperMajorityApprove,
delay: 1u32.into(),
tally: Tally { ayes: 1u32.into(), nays: 1u32.into(), turnout: 1u32.into() },
}))
);
// Case 2: Finished referendum
assert_eq!(
ReferendumInfoOf::<T>::get(2u32),
Some(ReferendumInfo::Finished { approved: true, end: 123u32.into() })
);
// Case 3: Public proposals
let props: BoundedVec<_, <Test as Config>::MaxProposals> = bounded_vec![
(3u32, Bounded::from_legacy_hash(hash), 123u64),
(4u32, Bounded::from_legacy_hash(hash2), 123u64)
];
assert_eq!(PublicProps::<T>::get(), props);
// Case 4: Next external
assert_eq!(
NextExternal::<T>::get(),
Some((Bounded::from_legacy_hash(hash), VoteThreshold::SuperMajorityApprove))
);
});
}
}