Refactor away from opaque hashes (#5226)

* System.BlockHash

* Fix hash

* Introduce K/V iteration in all _concat maps

Also move across:
- System.Account (blake2_128_concat)
- Balances.Locks (twox_64_concat)
- ElectionsPhragmen.VotesOf (twox_64_concat)
- ElectionsPhragmen.StakeOf (twox_64_concat)
- Identity.IdentityOf (twox_64_concat)
- Identity.SubsOf (twox_64_concat)
- Society.Payouts (twox_64_concat)
- Session.NextKeys (twox_64_concat)
- Identity.SuperOf (blake2_128_concat)
- Session.KeyOwner (blake2_128_concat)
- Society.SuspendedCandidates (twox_64_concat)
- Society.SuspendedMembers (twox_64_concat)
- Society.Vouching (twox_64_concat)
- Society.Strikes (twox_64_concat)
- System.EventTopics
- Balances.Account

* Build fixes

* Ensure migration happens in correct order

* Staking.*

* Vesting.* Offences.*

* Democracy.*

* Babe.* Collective.*

* Grandpa.*

* Assets.* Benchmark.* Contracts.* Elections.* Asset.* Nicks.*

Also introduce real account list

* ImOnline.*

* Treasury.*

* Recovery.*

* Final bits.

* Docs

* Fix one test

* Fix test

* All passing except the UI tests

* Remove linked_map part 1

* Remove linked_map

* Some iterator utils for double maps.

* Remove old migrations

* Introduce tombstone for LinkedMap type

* Migration for genesis hash

* Fix build

* Fix hash

* Rename Map is_linked -> unused, keeping backwards compat (#5256)

* Update frame/balances/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update frame/elections/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Remove old migration code.

* Update frame/system/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update bin/node/runtime/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Fix hash

* fix session migration

* Fix watning

Co-authored-by: Jaco Greeff <jacogr@gmail.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

substrate/frame/support/test/tests/decl_storage.rs

@@ -37,10 +37,10 @@ mod tests {
 			// non-getters: pub / $default
 
 			/// Hello, this is doc!
-			U32 : Option<u32>;
-			pub PUBU32 : Option<u32>;
-			U32MYDEF : Option<u32>;
-			pub PUBU32MYDEF : Option<u32>;
+			U32: Option<u32>;
+			pub PUBU32: Option<u32>;
+			U32MYDEF: Option<u32>;
+			pub PUBU32MYDEF: Option<u32>;
 
 			// getters: pub / $default
 			// we need at least one type which uses T, otherwise GenesisConfig will complain.
@@ -59,31 +59,23 @@ mod tests {
 			GetOptU32WithBuilderNone get(fn opt_u32_with_builder_none) build(|_| None): Option<u32>;
 
 			// map non-getters: pub / $default
-			MAPU32 : map hasher(blake2_256) u32 => Option<String>;
-			pub PUBMAPU32 : map hasher(blake2_256) u32 => Option<String>;
-			MAPU32MYDEF : map hasher(blake2_256) u32 => Option<String>;
-			pub PUBMAPU32MYDEF : map hasher(blake2_256) u32 => Option<String>;
+			MAPU32: map hasher(blake2_128_concat) u32 => Option<String>;
+			pub PUBMAPU32: map hasher(blake2_128_concat) u32 => Option<String>;
+			MAPU32MYDEF: map hasher(blake2_128_concat) u32 => Option<String>;
+			pub PUBMAPU32MYDEF: map hasher(blake2_128_concat) u32 => Option<String>;
 
 			// map getters: pub / $default
-			GETMAPU32 get(fn map_u32_getter): map hasher(blake2_256) u32 => String;
-			pub PUBGETMAPU32 get(fn pub_map_u32_getter): map hasher(blake2_256) u32 => String;
+			GETMAPU32 get(fn map_u32_getter): map hasher(blake2_128_concat) u32 => String;
+			pub PUBGETMAPU32 get(fn pub_map_u32_getter): map hasher(blake2_128_concat) u32 => String;
 
 			GETMAPU32MYDEF get(fn map_u32_getter_mydef):
-				map hasher(blake2_256) u32 => String = "map".into();
+				map hasher(blake2_128_concat) u32 => String = "map".into();
 			pub PUBGETMAPU32MYDEF get(fn pub_map_u32_getter_mydef):
-				map hasher(blake2_256) u32 => String = "pubmap".into();
-
-			// linked map
-			LINKEDMAPU32 : linked_map hasher(blake2_256) u32 => Option<String>;
-			pub PUBLINKEDMAPU32MYDEF : linked_map hasher(blake2_256) u32 => Option<String>;
-			GETLINKEDMAPU32 get(fn linked_map_u32_getter):
-				linked_map hasher(blake2_256) u32 => String;
-			pub PUBGETLINKEDMAPU32MYDEF get(fn pub_linked_map_u32_getter_mydef):
-				linked_map hasher(blake2_256) u32 => String = "pubmap".into();
+				map hasher(blake2_128_concat) u32 => String = "pubmap".into();
 
 			COMPLEXTYPE1: ::std::vec::Vec<<T as Trait>::Origin>;
-			COMPLEXTYPE2: (Vec<Vec<(u16,Box<(  )>)>>, u32);
-			COMPLEXTYPE3: [u32;25];
+			COMPLEXTYPE2: (Vec<Vec<(u16, Box<()>)>>, u32);
+			COMPLEXTYPE3: [u32; 25];
 		}
 		add_extra_genesis {
 			build(|_| {});
@@ -249,10 +241,10 @@ mod tests {
 					name: DecodeDifferent::Encode("MAPU32"),
 					modifier: StorageEntryModifier::Optional,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructMAPU32(PhantomData::<TraitImpl>))
@@ -263,10 +255,10 @@ mod tests {
 					name: DecodeDifferent::Encode("PUBMAPU32"),
 					modifier: StorageEntryModifier::Optional,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructPUBMAPU32(PhantomData::<TraitImpl>))
@@ -277,10 +269,10 @@ mod tests {
 					name: DecodeDifferent::Encode("MAPU32MYDEF"),
 					modifier: StorageEntryModifier::Optional,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructMAPU32MYDEF(PhantomData::<TraitImpl>))
@@ -291,10 +283,10 @@ mod tests {
 					name: DecodeDifferent::Encode("PUBMAPU32MYDEF"),
 					modifier: StorageEntryModifier::Optional,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructPUBMAPU32MYDEF(PhantomData::<TraitImpl>))
@@ -305,10 +297,10 @@ mod tests {
 					name: DecodeDifferent::Encode("GETMAPU32"),
 					modifier: StorageEntryModifier::Default,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructGETMAPU32(PhantomData::<TraitImpl>))
@@ -319,10 +311,10 @@ mod tests {
 					name: DecodeDifferent::Encode("PUBGETMAPU32"),
 					modifier: StorageEntryModifier::Default,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructPUBGETMAPU32(PhantomData::<TraitImpl>))
@@ -333,10 +325,10 @@ mod tests {
 					name: DecodeDifferent::Encode("GETMAPU32MYDEF"),
 					modifier: StorageEntryModifier::Default,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructGETMAPU32MYDEF(PhantomData::<TraitImpl>))
@@ -347,72 +339,16 @@ mod tests {
 					name: DecodeDifferent::Encode("PUBGETMAPU32MYDEF"),
 					modifier: StorageEntryModifier::Default,
 					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
+						hasher: StorageHasher::Blake2_128Concat,
 						key: DecodeDifferent::Encode("u32"),
 						value: DecodeDifferent::Encode("String"),
-						is_linked: false,
+						unused: false,
 					},
 					default: DecodeDifferent::Encode(
 						DefaultByteGetter(&__GetByteStructPUBGETMAPU32MYDEF(PhantomData::<TraitImpl>))
 					),
 					documentation: DecodeDifferent::Encode(&[]),
 				},
-				StorageEntryMetadata {
-					name: DecodeDifferent::Encode("LINKEDMAPU32"),
-					modifier: StorageEntryModifier::Optional,
-					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
-						key: DecodeDifferent::Encode("u32"),
-						value: DecodeDifferent::Encode("String"),
-						is_linked: true,
-					},
-					default: DecodeDifferent::Encode(
-						DefaultByteGetter(&__GetByteStructLINKEDMAPU32(PhantomData::<TraitImpl>))
-					),
-					documentation: DecodeDifferent::Encode(&[]),
-				},
-				StorageEntryMetadata {
-					name: DecodeDifferent::Encode("PUBLINKEDMAPU32MYDEF"),
-					modifier: StorageEntryModifier::Optional,
-					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
-						key: DecodeDifferent::Encode("u32"),
-						value: DecodeDifferent::Encode("String"),
-						is_linked: true,
-					},
-					default: DecodeDifferent::Encode(
-						DefaultByteGetter(&__GetByteStructPUBLINKEDMAPU32MYDEF(PhantomData::<TraitImpl>))
-					),
-					documentation: DecodeDifferent::Encode(&[]),
-				},
-				StorageEntryMetadata {
-					name: DecodeDifferent::Encode("GETLINKEDMAPU32"),
-					modifier: StorageEntryModifier::Default,
-					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
-						key: DecodeDifferent::Encode("u32"),
-						value: DecodeDifferent::Encode("String"),
-						is_linked: true,
-					},
-					default: DecodeDifferent::Encode(
-						DefaultByteGetter(&__GetByteStructGETLINKEDMAPU32(PhantomData::<TraitImpl>))
-					),
-					documentation: DecodeDifferent::Encode(&[]),
-				},
-				StorageEntryMetadata {
-					name: DecodeDifferent::Encode("PUBGETLINKEDMAPU32MYDEF"),
-					modifier: StorageEntryModifier::Default,
-					ty: StorageEntryType::Map {
-						hasher: StorageHasher::Blake2_256,
-						key: DecodeDifferent::Encode("u32"),
-						value: DecodeDifferent::Encode("String"),
-						is_linked: true,
-					},
-					default: DecodeDifferent::Encode(
-						DefaultByteGetter(&__GetByteStructPUBGETLINKEDMAPU32MYDEF(PhantomData::<TraitImpl>))
-					),
-					documentation: DecodeDifferent::Encode(&[]),
-				},
 				StorageEntryMetadata {
 					name: DecodeDifferent::Encode("COMPLEXTYPE1"),
 					modifier: StorageEntryModifier::Default,
@@ -562,17 +498,13 @@ mod test_append_and_len {
 			JustVecWithDefault: Vec<u32> = vec![6, 9];
 			OptionVec: Option<Vec<u32>>;
 
-			MapVec: map hasher(blake2_256) u32 => Vec<u32>;
-			MapVecWithDefault: map hasher(blake2_256) u32 => Vec<u32> = vec![6, 9];
-			OptionMapVec: map hasher(blake2_256) u32 => Option<Vec<u32>>;
+			MapVec: map hasher(blake2_128_concat) u32 => Vec<u32>;
+			MapVecWithDefault: map hasher(blake2_128_concat) u32 => Vec<u32> = vec![6, 9];
+			OptionMapVec: map hasher(blake2_128_concat) u32 => Option<Vec<u32>>;
 
-			DoubleMapVec: double_map hasher(blake2_256) u32, hasher(blake2_256) u32 => Vec<u32>;
-			DoubleMapVecWithDefault: double_map hasher(blake2_256) u32, hasher(blake2_256) u32 => Vec<u32> = vec![6, 9];
-			OptionDoubleMapVec: double_map hasher(blake2_256) u32, hasher(blake2_256) u32 => Option<Vec<u32>>;
-
-			LinkedMapVec: linked_map hasher(blake2_256) u32 => Vec<u32>;
-			LinkedMapVecWithDefault: linked_map hasher(blake2_256) u32 => Vec<u32> = vec![6, 9];
-			OptionLinkedMapVec: linked_map hasher(blake2_256) u32 => Option<Vec<u32>>;
+			DoubleMapVec: double_map hasher(blake2_128_concat) u32, hasher(blake2_128_concat) u32 => Vec<u32>;
+			DoubleMapVecWithDefault: double_map hasher(blake2_128_concat) u32, hasher(blake2_128_concat) u32 => Vec<u32> = vec![6, 9];
+			OptionDoubleMapVec: double_map hasher(blake2_128_concat) u32, hasher(blake2_128_concat) u32 => Option<Vec<u32>>;
 		}
 	}
 
@@ -644,13 +576,11 @@ mod test_append_and_len {
 			JustVec::put(&vec![1, 2, 3, 4]);
 			OptionVec::put(&vec![1, 2, 3, 4, 5]);
 			MapVec::insert(1, &vec![1, 2, 3, 4, 5, 6]);
-			LinkedMapVec::insert(2, &vec![1, 2, 3]);
 			DoubleMapVec::insert(0, 1, &vec![1, 2]);
 
 			assert_eq!(JustVec::decode_len().unwrap(), 4);
 			assert_eq!(OptionVec::decode_len().unwrap(), 5);
 			assert_eq!(MapVec::decode_len(1).unwrap(), 6);
-			assert_eq!(LinkedMapVec::decode_len(2).unwrap(), 3);
 			assert_eq!(DoubleMapVec::decode_len(0, 1).unwrap(), 2);
 		});
 	}
@@ -678,16 +608,6 @@ mod test_append_and_len {
 			assert_eq!(OptionMapVec::get(0), None);
 			assert_eq!(OptionMapVec::decode_len(0), Ok(0));
 
-			// linked map
-			assert_eq!(LinkedMapVec::get(0), vec![]);
-			assert_eq!(LinkedMapVec::decode_len(0), Ok(0));
-
-			assert_eq!(LinkedMapVecWithDefault::get(0), vec![6, 9]);
-			assert_eq!(LinkedMapVecWithDefault::decode_len(0), Ok(2));
-
-			assert_eq!(OptionLinkedMapVec::get(0), None);
-			assert_eq!(OptionLinkedMapVec::decode_len(0), Ok(0));
-
 			// Double map
 			assert_eq!(DoubleMapVec::get(0, 0), vec![]);
 			assert_eq!(DoubleMapVec::decode_len(0, 1), Ok(0));