Refactor away from opaque hashes (#5226)

* System.BlockHash

* Fix hash

* Introduce K/V iteration in all _concat maps

Also move across:
- System.Account (blake2_128_concat)
- Balances.Locks (twox_64_concat)
- ElectionsPhragmen.VotesOf (twox_64_concat)
- ElectionsPhragmen.StakeOf (twox_64_concat)
- Identity.IdentityOf (twox_64_concat)
- Identity.SubsOf (twox_64_concat)
- Society.Payouts (twox_64_concat)
- Session.NextKeys (twox_64_concat)
- Identity.SuperOf (blake2_128_concat)
- Session.KeyOwner (blake2_128_concat)
- Society.SuspendedCandidates (twox_64_concat)
- Society.SuspendedMembers (twox_64_concat)
- Society.Vouching (twox_64_concat)
- Society.Strikes (twox_64_concat)
- System.EventTopics
- Balances.Account

* Build fixes

* Ensure migration happens in correct order

* Staking.*

* Vesting.* Offences.*

* Democracy.*

* Babe.* Collective.*

* Grandpa.*

* Assets.* Benchmark.* Contracts.* Elections.* Asset.* Nicks.*

Also introduce real account list

* ImOnline.*

* Treasury.*

* Recovery.*

* Final bits.

* Docs

* Fix one test

* Fix test

* All passing except the UI tests

* Remove linked_map part 1

* Remove linked_map

* Some iterator utils for double maps.

* Remove old migrations

* Introduce tombstone for LinkedMap type

* Migration for genesis hash

* Fix build

* Fix hash

* Rename Map is_linked -> unused, keeping backwards compat (#5256)

* Update frame/balances/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update frame/elections/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Remove old migration code.

* Update frame/system/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update bin/node/runtime/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Fix hash

* fix session migration

* Fix watning

Co-authored-by: Jaco Greeff <jacogr@gmail.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

substrate/frame/session/src/historical.rs

@@ -56,12 +56,12 @@ decl_storage! {
 	trait Store for Module<T: Trait> as Session {
 		/// Mapping from historical session indices to session-data root hash and validator count.
 		HistoricalSessions get(fn historical_root):
-			map hasher(blake2_256) SessionIndex => Option<(T::Hash, ValidatorCount)>;
+			map hasher(twox_64_concat) SessionIndex => Option<(T::Hash, ValidatorCount)>;
 		/// The range of historical sessions we store. [first, last)
 		StoredRange: Option<(SessionIndex, SessionIndex)>;
 		/// Deprecated.
 		CachedObsolete:
-			map hasher(blake2_256) SessionIndex
+			map hasher(twox_64_concat) SessionIndex
 			=> Option<Vec<(T::ValidatorId, T::FullIdentification)>>;
 	}
 }
@@ -71,6 +71,20 @@ decl_module! {
 		fn on_initialize(_n: T::BlockNumber) {
 			CachedObsolete::<T>::remove_all();
 		}
+		fn on_runtime_upgrade() {
+			migration::migrate::<T>();
+		}
+	}
+}
+
+mod migration {
+	use super::*;
+	pub fn migrate<T: Trait>() {
+		if let Some((begin, end)) = StoredRange::get() {
+			for i in begin..end {
+				HistoricalSessions::<T>::migrate_key_from_blake(i);
+			}
+		}
 	}
 }
 

substrate/frame/session/src/lib.rs

@@ -109,6 +109,7 @@ use frame_support::{ensure, decl_module, decl_event, decl_storage, decl_error, C
 use frame_support::{traits::{Get, FindAuthor, ValidatorRegistration}, Parameter};
 use frame_support::dispatch::{self, DispatchResult, DispatchError};
 use frame_system::{self as system, ensure_signed};
+use frame_support::traits::MigrateAccount;
 
 #[cfg(test)]
 mod mock;
@@ -364,10 +365,10 @@ decl_storage! {
 		DisabledValidators get(fn disabled_validators): Vec<u32>;
 
 		/// The next session keys for a validator.
-		NextKeys: map hasher(blake2_256) T::ValidatorId => Option<T::Keys>;
+		NextKeys: map hasher(twox_64_concat) T::ValidatorId => Option<T::Keys>;
 
 		/// The owner of a key. The key is the `KeyTypeId` + the encoded key.
-		KeyOwner: map hasher(blake2_256) (KeyTypeId, Vec<u8>) => Option<T::ValidatorId>;
+		KeyOwner: map hasher(twox_64_concat) (KeyTypeId, Vec<u8>) => Option<T::ValidatorId>;
 	}
 	add_extra_genesis {
 		config(keys): Vec<(T::AccountId, T::ValidatorId, T::Keys)>;
@@ -498,36 +499,22 @@ decl_module! {
 				Self::rotate_session();
 			}
 		}
+	}
+}
 
-		/// Called when the runtime is upgraded.
-		fn on_runtime_upgrade() {
-			Self::migrate();
+impl<T: Trait> MigrateAccount<T::AccountId> for Module<T> {
+	fn migrate_account(a: &T::AccountId) {
+		if let Some(v) = T::ValidatorIdOf::convert(a.clone()) {
+			if let Some(keys) = NextKeys::<T>::migrate_key_from_blake(v) {
+				for id in T::Keys::key_ids() {
+					KeyOwner::<T>::migrate_key_from_blake((*id, keys.get_raw(*id)));
+				}
+			}
 		}
 	}
 }
 
 impl<T: Trait> Module<T> {
-	/// Move keys from NextKeys and KeyOwner, if any exist.
-	fn migrate() {
-		use frame_support::storage::migration::{put_storage_value, StorageIterator};
-		sp_runtime::print("Migrating session's double-maps...");
-
-		let prefix = {
-			const DEDUP_KEY_PREFIX: &[u8] = b":session:keys";
-			let encoded_prefix_key_hash = codec::Encode::encode(&DEDUP_KEY_PREFIX);
-			let mut h = sp_io::hashing::twox_64(&encoded_prefix_key_hash[..]).to_vec();
-			h.extend(&encoded_prefix_key_hash[..]);
-			h
-		};
-
-		for (hash, value) in StorageIterator::<T::Keys>::with_suffix(b"Session", b"NextKeys", &prefix[..]).drain() {
-			put_storage_value(b"Session", b"NextKeys", &hash, value);
-		}
-		for (hash, value) in StorageIterator::<T::ValidatorId>::with_suffix(b"Session", b"KeyOwner", &prefix[..]).drain() {
-			put_storage_value(b"Session", b"KeyOwner", &hash, value);
-		}
-	}
-
 	/// Move on to next session. Register new validator set and session keys. Changes
 	/// to the validator set have a session of delay to take effect. This allows for
 	/// equivocation punishment after a fork.

substrate/frame/session/src/mock.rs

@@ -179,7 +179,7 @@ impl frame_system::Trait for Test {
 	type Version = ();
 	type ModuleToIndex = ();
 	type AccountData = ();
-	type OnNewAccount = ();
+	type MigrateAccount = (); type OnNewAccount = ();
 	type OnKilledAccount = ();
 }