pallet-session: Migrate the historical part to the new pallet macro (#9878)

* Migrate session-historical to the new pallet macro

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* pallet-session: Migrate the historical part to the new pallet macro

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* Fix staking test runtime

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* Update frame/session/src/historical/mod.rs

* Update frame/session/src/historical/mod.rs

* update migration doc

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* use hardcoded prefix for migration v1

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* cargo +nightly-2021-11-08 fmt

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

Co-authored-by: Guillaume Thiolliere <gui.thiolliere@gmail.com>
This commit is contained in:
Qinxuan Chen
2021-11-17 15:16:28 +08:00
committed by GitHub
parent 256c35f473
commit 0b224d1b60
8 changed files with 325 additions and 66 deletions
@@ -22,6 +22,7 @@
mod mock;
use sp_runtime::traits::{One, StaticLookup};
use sp_std::{prelude::*, vec};
use frame_benchmarking::benchmarks;
@@ -30,12 +31,11 @@ use frame_support::{
traits::{KeyOwnerProofSystem, OnInitialize},
};
use frame_system::RawOrigin;
use pallet_session::{historical::Module as Historical, Pallet as Session, *};
use pallet_session::{historical::Pallet as Historical, Pallet as Session, *};
use pallet_staking::{
benchmarking::create_validator_with_nominators, testing_utils::create_validators,
RewardDestination,
};
use sp_runtime::traits::{One, StaticLookup};
const MAX_VALIDATORS: u32 = 1000;
+74 -62
View File
@@ -26,62 +26,74 @@
//! These roots and proofs of inclusion can be generated at any time during the current session.
//! Afterwards, the proofs can be fed to a consensus module when reporting misbehavior.
use super::{Pallet as SessionModule, SessionIndex};
pub mod offchain;
pub mod onchain;
mod shared;
use codec::{Decode, Encode};
use frame_support::{
decl_module, decl_storage, print,
traits::{ValidatorSet, ValidatorSetWithIdentification},
Parameter,
};
use sp_runtime::{
traits::{Convert, OpaqueKeys},
KeyTypeId,
};
use sp_session::{MembershipProof, ValidatorCount};
use sp_staking::SessionIndex;
use sp_std::prelude::*;
use sp_trie::{
trie_types::{TrieDB, TrieDBMut},
MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX,
};
pub mod offchain;
pub mod onchain;
mod shared;
use frame_support::{
print,
traits::{KeyOwnerProofSystem, StorageVersion, ValidatorSet, ValidatorSetWithIdentification},
Parameter,
};
/// Config necessary for the historical module.
pub trait Config: super::Config {
/// Full identification of the validator.
type FullIdentification: Parameter;
use crate::{self as pallet_session, Pallet as Session};
/// A conversion from validator ID to full identification.
///
/// This should contain any references to economic actors associated with the
/// validator, since they may be outdated by the time this is queried from a
/// historical trie.
///
/// It must return the identification for the current session index.
type FullIdentificationOf: Convert<Self::ValidatorId, Option<Self::FullIdentification>>;
}
pub use pallet::*;
decl_storage! {
trait Store for Module<T: Config> as Session {
/// Mapping from historical session indices to session-data root hash and validator count.
HistoricalSessions get(fn historical_root):
map hasher(twox_64_concat) SessionIndex => Option<(T::Hash, ValidatorCount)>;
/// The range of historical sessions we store. [first, last)
StoredRange: Option<(SessionIndex, SessionIndex)>;
/// Deprecated.
CachedObsolete:
map hasher(twox_64_concat) SessionIndex
=> Option<Vec<(T::ValidatorId, T::FullIdentification)>>;
#[frame_support::pallet]
pub mod pallet {
use super::*;
use frame_support::pallet_prelude::*;
/// The current storage version.
const STORAGE_VERSION: StorageVersion = StorageVersion::new(1);
#[pallet::pallet]
#[pallet::generate_store(pub(super) trait Store)]
#[pallet::storage_version(STORAGE_VERSION)]
pub struct Pallet<T>(_);
/// Config necessary for the historical pallet.
#[pallet::config]
pub trait Config: pallet_session::Config + frame_system::Config {
/// Full identification of the validator.
type FullIdentification: Parameter;
/// A conversion from validator ID to full identification.
///
/// This should contain any references to economic actors associated with the
/// validator, since they may be outdated by the time this is queried from a
/// historical trie.
///
/// It must return the identification for the current session index.
type FullIdentificationOf: Convert<Self::ValidatorId, Option<Self::FullIdentification>>;
}
/// Mapping from historical session indices to session-data root hash and validator count.
#[pallet::storage]
#[pallet::getter(fn historical_root)]
pub type HistoricalSessions<T: Config> =
StorageMap<_, Twox64Concat, SessionIndex, (T::Hash, ValidatorCount), OptionQuery>;
/// The range of historical sessions we store. [first, last)
#[pallet::storage]
pub type StoredRange<T> = StorageValue<_, (SessionIndex, SessionIndex), OptionQuery>;
}
decl_module! {
pub struct Module<T: Config> for enum Call where origin: T::Origin {}
}
impl<T: Config> Module<T> {
impl<T: Config> Pallet<T> {
/// Prune historical stored session roots up to (but not including)
/// `up_to`.
pub fn prune_up_to(up_to: SessionIndex) {
@@ -109,7 +121,7 @@ impl<T: Config> Module<T> {
}
}
impl<T: Config> ValidatorSet<T::AccountId> for Module<T> {
impl<T: Config> ValidatorSet<T::AccountId> for Pallet<T> {
type ValidatorId = T::ValidatorId;
type ValidatorIdOf = T::ValidatorIdOf;
@@ -122,7 +134,7 @@ impl<T: Config> ValidatorSet<T::AccountId> for Module<T> {
}
}
impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Module<T> {
impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Pallet<T> {
type Identification = T::FullIdentification;
type IdentificationOf = T::FullIdentificationOf;
}
@@ -130,7 +142,7 @@ impl<T: Config> ValidatorSetWithIdentification<T::AccountId> for Module<T> {
/// Specialization of the crate-level `SessionManager` which returns the set of full identification
/// when creating a new session.
pub trait SessionManager<ValidatorId, FullIdentification>:
crate::SessionManager<ValidatorId>
pallet_session::SessionManager<ValidatorId>
{
/// If there was a validator set change, its returns the set of new validators along with their
/// full identifications.
@@ -150,7 +162,7 @@ pub struct NoteHistoricalRoot<T, I>(sp_std::marker::PhantomData<(T, I)>);
impl<T: Config, I: SessionManager<T::ValidatorId, T::FullIdentification>> NoteHistoricalRoot<T, I> {
fn do_new_session(new_index: SessionIndex, is_genesis: bool) -> Option<Vec<T::ValidatorId>> {
StoredRange::mutate(|range| {
<StoredRange<T>>::mutate(|range| {
range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1;
});
@@ -183,7 +195,7 @@ impl<T: Config, I: SessionManager<T::ValidatorId, T::FullIdentification>> NoteHi
}
}
impl<T: Config, I> crate::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T, I>
impl<T: Config, I> pallet_session::SessionManager<T::ValidatorId> for NoteHistoricalRoot<T, I>
where
I: SessionManager<T::ValidatorId, T::FullIdentification>,
{
@@ -207,7 +219,7 @@ where
/// A tuple of the validator's ID and their full identification.
pub type IdentificationTuple<T> =
(<T as crate::Config>::ValidatorId, <T as Config>::FullIdentification);
(<T as pallet_session::Config>::ValidatorId, <T as Config>::FullIdentification);
/// A trie instance for checking and generating proofs.
pub struct ProvingTrie<T: Config> {
@@ -227,7 +239,7 @@ impl<T: Config> ProvingTrie<T> {
let mut trie = TrieDBMut::new(&mut db, &mut root);
for (i, (validator, full_id)) in validators.into_iter().enumerate() {
let i = i as u32;
let keys = match <SessionModule<T>>::load_keys(&validator) {
let keys = match <Session<T>>::load_keys(&validator) {
None => continue,
Some(k) => k,
};
@@ -304,15 +316,13 @@ impl<T: Config> ProvingTrie<T> {
}
}
impl<T: Config, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)>
for Module<T>
{
impl<T: Config, D: AsRef<[u8]>> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet<T> {
type Proof = MembershipProof;
type IdentificationTuple = IdentificationTuple<T>;
fn prove(key: (KeyTypeId, D)) -> Option<Self::Proof> {
let session = <SessionModule<T>>::current_index();
let validators = <SessionModule<T>>::validators()
let session = <Session<T>>::current_index();
let validators = <Session<T>>::validators()
.into_iter()
.filter_map(|validator| {
T::FullIdentificationOf::convert(validator.clone())
@@ -335,10 +345,10 @@ impl<T: Config, D: AsRef<[u8]>> frame_support::traits::KeyOwnerProofSystem<(KeyT
fn check_proof(key: (KeyTypeId, D), proof: Self::Proof) -> Option<IdentificationTuple<T>> {
let (id, data) = key;
if proof.session == <SessionModule<T>>::current_index() {
<SessionModule<T>>::key_owner(id, data.as_ref()).and_then(|owner| {
if proof.session == <Session<T>>::current_index() {
<Session<T>>::key_owner(id, data.as_ref()).and_then(|owner| {
T::FullIdentificationOf::convert(owner.clone()).and_then(move |id| {
let count = <SessionModule<T>>::validators().len() as ValidatorCount;
let count = <Session<T>>::validators().len() as ValidatorCount;
if count != proof.validator_count {
return None
@@ -374,7 +384,7 @@ pub(crate) mod tests {
BasicExternalities,
};
type Historical = Module<Test>;
type Historical = Pallet<Test>;
pub(crate) fn new_test_ext() -> sp_io::TestExternalities {
let mut t = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();
@@ -386,7 +396,9 @@ pub(crate) mod tests {
frame_system::Pallet::<Test>::inc_providers(k);
}
});
crate::GenesisConfig::<Test> { keys }.assimilate_storage(&mut t).unwrap();
pallet_session::GenesisConfig::<Test> { keys }
.assimilate_storage(&mut t)
.unwrap();
sp_io::TestExternalities::new(t)
}
@@ -436,27 +448,27 @@ pub(crate) mod tests {
Session::on_initialize(i);
}
assert_eq!(StoredRange::get(), Some((0, 100)));
assert_eq!(<StoredRange<Test>>::get(), Some((0, 100)));
for i in 0..100 {
assert!(Historical::historical_root(i).is_some())
}
Historical::prune_up_to(10);
assert_eq!(StoredRange::get(), Some((10, 100)));
assert_eq!(<StoredRange<Test>>::get(), Some((10, 100)));
Historical::prune_up_to(9);
assert_eq!(StoredRange::get(), Some((10, 100)));
assert_eq!(<StoredRange<Test>>::get(), Some((10, 100)));
for i in 10..100 {
assert!(Historical::historical_root(i).is_some())
}
Historical::prune_up_to(99);
assert_eq!(StoredRange::get(), Some((99, 100)));
assert_eq!(<StoredRange<Test>>::get(), Some((99, 100)));
Historical::prune_up_to(100);
assert_eq!(StoredRange::get(), None);
assert_eq!(<StoredRange<Test>>::get(), None);
for i in 99..199u64 {
set_next_validators(vec![i]);
@@ -466,14 +478,14 @@ pub(crate) mod tests {
Session::on_initialize(i);
}
assert_eq!(StoredRange::get(), Some((100, 200)));
assert_eq!(<StoredRange<Test>>::get(), Some((100, 200)));
for i in 100..200 {
assert!(Historical::historical_root(i).is_some())
}
Historical::prune_up_to(9999);
assert_eq!(StoredRange::get(), None);
assert_eq!(<StoredRange<Test>>::get(), None);
for i in 100..200 {
assert!(Historical::historical_root(i).is_none())
@@ -140,7 +140,7 @@ pub fn keep_newest<T: Config>(n_to_keep: usize) {
mod tests {
use super::*;
use crate::{
historical::{onchain, Module},
historical::{onchain, Pallet},
mock::{force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS},
};
@@ -156,7 +156,7 @@ mod tests {
BasicExternalities,
};
type Historical = Module<Test>;
type Historical = Pallet<Test>;
pub fn new_test_ext() -> sp_io::TestExternalities {
let mut t = frame_system::GenesisConfig::default()
+1
View File
@@ -108,6 +108,7 @@
#[cfg(feature = "historical")]
pub mod historical;
pub mod migrations;
#[cfg(test)]
mod mock;
#[cfg(test)]
@@ -0,0 +1,24 @@
// This file is part of Substrate.
// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// Version 1.
///
/// In version 0 session historical pallet uses `Session` for storage module prefix.
/// In version 1 it uses its name as configured in `construct_runtime`.
/// This migration moves session historical pallet storages from old prefix to new prefix.
#[cfg(feature = "historical")]
pub mod v1;
@@ -0,0 +1,194 @@
// This file is part of Substrate.
// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use sp_io::hashing::twox_128;
use sp_std::str;
use frame_support::{
storage::{generator::StorageValue, StoragePrefixedMap},
traits::{
Get, GetStorageVersion, PalletInfoAccess, StorageVersion,
STORAGE_VERSION_STORAGE_KEY_POSTFIX,
},
weights::Weight,
};
use crate::historical as pallet_session_historical;
const OLD_PREFIX: &str = "Session";
/// Migrate the entire storage of this pallet to a new prefix.
///
/// This new prefix must be the same as the one set in construct_runtime.
///
/// The migration will look into the storage version in order not to trigger a migration on an up
/// to date storage. Thus the on chain storage version must be less than 1 in order to trigger the
/// migration.
pub fn migrate<T: pallet_session_historical::Config, P: GetStorageVersion + PalletInfoAccess>(
) -> Weight {
let new_pallet_name = <P as PalletInfoAccess>::name();
if new_pallet_name == OLD_PREFIX {
log::info!(
target: "runtime::session_historical",
"New pallet name is equal to the old prefix. No migration needs to be done.",
);
return 0
}
let on_chain_storage_version = <P as GetStorageVersion>::on_chain_storage_version();
log::info!(
target: "runtime::session_historical",
"Running migration to v1 for session_historical with storage version {:?}",
on_chain_storage_version,
);
if on_chain_storage_version < 1 {
let storage_prefix = pallet_session_historical::HistoricalSessions::<T>::storage_prefix();
frame_support::storage::migration::move_storage_from_pallet(
storage_prefix,
OLD_PREFIX.as_bytes(),
new_pallet_name.as_bytes(),
);
log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name);
let storage_prefix = pallet_session_historical::StoredRange::<T>::storage_prefix();
frame_support::storage::migration::move_storage_from_pallet(
storage_prefix,
OLD_PREFIX.as_bytes(),
new_pallet_name.as_bytes(),
);
log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name);
StorageVersion::new(1).put::<P>();
<T as frame_system::Config>::BlockWeights::get().max_block
} else {
log::warn!(
target: "runtime::session_historical",
"Attempted to apply migration to v1 but failed because storage version is {:?}",
on_chain_storage_version,
);
0
}
}
/// Some checks prior to migration. This can be linked to
/// [`frame_support::traits::OnRuntimeUpgrade::pre_upgrade`] for further testing.
///
/// Panics if anything goes wrong.
pub fn pre_migrate<
T: pallet_session_historical::Config,
P: GetStorageVersion + PalletInfoAccess,
>() {
let new_pallet_name = <P as PalletInfoAccess>::name();
let storage_prefix_historical_sessions =
pallet_session_historical::HistoricalSessions::<T>::storage_prefix();
let storage_prefix_stored_range = pallet_session_historical::StoredRange::<T>::storage_prefix();
log_migration("pre-migration", storage_prefix_historical_sessions, OLD_PREFIX, new_pallet_name);
log_migration("pre-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name);
if new_pallet_name == OLD_PREFIX {
return
}
let new_pallet_prefix = twox_128(new_pallet_name.as_bytes());
let storage_version_key = twox_128(STORAGE_VERSION_STORAGE_KEY_POSTFIX);
let mut new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new(
new_pallet_prefix.to_vec(),
new_pallet_prefix.to_vec(),
|key| Ok(key.to_vec()),
);
// Ensure nothing except the storage_version_key is stored in the new prefix.
assert!(new_pallet_prefix_iter.all(|key| key == storage_version_key));
assert!(<P as GetStorageVersion>::on_chain_storage_version() < 1);
}
/// Some checks for after migration. This can be linked to
/// [`frame_support::traits::OnRuntimeUpgrade::post_upgrade`] for further testing.
///
/// Panics if anything goes wrong.
pub fn post_migrate<
T: pallet_session_historical::Config,
P: GetStorageVersion + PalletInfoAccess,
>() {
let new_pallet_name = <P as PalletInfoAccess>::name();
let storage_prefix_historical_sessions =
pallet_session_historical::HistoricalSessions::<T>::storage_prefix();
let storage_prefix_stored_range = pallet_session_historical::StoredRange::<T>::storage_prefix();
log_migration(
"post-migration",
storage_prefix_historical_sessions,
OLD_PREFIX,
new_pallet_name,
);
log_migration("post-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name);
if new_pallet_name == OLD_PREFIX {
return
}
// Assert that no `HistoricalSessions` and `StoredRange` storages remains at the old prefix.
let old_pallet_prefix = twox_128(OLD_PREFIX.as_bytes());
let old_historical_sessions_key =
[&old_pallet_prefix, &twox_128(storage_prefix_historical_sessions)[..]].concat();
let old_historical_sessions_key_iter = frame_support::storage::KeyPrefixIterator::new(
old_historical_sessions_key.to_vec(),
old_historical_sessions_key.to_vec(),
|_| Ok(()),
);
assert_eq!(old_historical_sessions_key_iter.count(), 0);
let old_stored_range_key =
[&old_pallet_prefix, &twox_128(storage_prefix_stored_range)[..]].concat();
let old_stored_range_key_iter = frame_support::storage::KeyPrefixIterator::new(
old_stored_range_key.to_vec(),
old_stored_range_key.to_vec(),
|_| Ok(()),
);
assert_eq!(old_stored_range_key_iter.count(), 0);
// Assert that the `HistoricalSessions` and `StoredRange` storages (if they exist) have been
// moved to the new prefix.
// NOTE: storage_version_key is already in the new prefix.
let new_pallet_prefix = twox_128(new_pallet_name.as_bytes());
let new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new(
new_pallet_prefix.to_vec(),
new_pallet_prefix.to_vec(),
|_| Ok(()),
);
assert!(new_pallet_prefix_iter.count() >= 1);
assert_eq!(<P as GetStorageVersion>::on_chain_storage_version(), 1);
}
fn log_migration(stage: &str, storage_prefix: &[u8], old_pallet_name: &str, new_pallet_name: &str) {
log::info!(
target: "runtime::session_historical",
"{} prefix of storage '{}': '{}' ==> '{}'",
stage,
str::from_utf8(storage_prefix).unwrap_or("<Invalid UTF8>"),
old_pallet_name,
new_pallet_name,
);
}
+27
View File
@@ -453,3 +453,30 @@ fn upgrade_keys() {
}
})
}
#[cfg(feature = "historical")]
#[test]
fn test_migration_v1() {
use crate::{
historical::{HistoricalSessions, StoredRange},
mock::Historical,
};
use frame_support::traits::PalletInfoAccess;
new_test_ext().execute_with(|| {
assert!(<HistoricalSessions<Test>>::iter_values().count() > 0);
assert!(<StoredRange<Test>>::exists());
let old_pallet = "Session";
let new_pallet = <Historical as PalletInfoAccess>::name();
frame_support::storage::migration::move_pallet(
new_pallet.as_bytes(),
old_pallet.as_bytes(),
);
StorageVersion::new(0).put::<Historical>();
crate::migrations::v1::pre_migrate::<Test, Historical>();
crate::migrations::v1::migrate::<Test, Historical>();
crate::migrations::v1::post_migrate::<Test, Historical>();
});
}
+1
View File
@@ -96,6 +96,7 @@ frame_support::construct_runtime!(
Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>},
Historical: pallet_session::historical::{Pallet, Storage},
BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
}
);