Refactor away from opaque hashes (#5226)

* System.BlockHash

* Fix hash

* Introduce K/V iteration in all _concat maps

Also move across:
- System.Account (blake2_128_concat)
- Balances.Locks (twox_64_concat)
- ElectionsPhragmen.VotesOf (twox_64_concat)
- ElectionsPhragmen.StakeOf (twox_64_concat)
- Identity.IdentityOf (twox_64_concat)
- Identity.SubsOf (twox_64_concat)
- Society.Payouts (twox_64_concat)
- Session.NextKeys (twox_64_concat)
- Identity.SuperOf (blake2_128_concat)
- Session.KeyOwner (blake2_128_concat)
- Society.SuspendedCandidates (twox_64_concat)
- Society.SuspendedMembers (twox_64_concat)
- Society.Vouching (twox_64_concat)
- Society.Strikes (twox_64_concat)
- System.EventTopics
- Balances.Account

* Build fixes

* Ensure migration happens in correct order

* Staking.*

* Vesting.* Offences.*

* Democracy.*

* Babe.* Collective.*

* Grandpa.*

* Assets.* Benchmark.* Contracts.* Elections.* Asset.* Nicks.*

Also introduce real account list

* ImOnline.*

* Treasury.*

* Recovery.*

* Final bits.

* Docs

* Fix one test

* Fix test

* All passing except the UI tests

* Remove linked_map part 1

* Remove linked_map

* Some iterator utils for double maps.

* Remove old migrations

* Introduce tombstone for LinkedMap type

* Migration for genesis hash

* Fix build

* Fix hash

* Rename Map is_linked -> unused, keeping backwards compat (#5256)

* Update frame/balances/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update frame/elections/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Remove old migration code.

* Update frame/system/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update bin/node/runtime/src/lib.rs

Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>

* Fix hash

* fix session migration

* Fix watning

Co-authored-by: Jaco Greeff <jacogr@gmail.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

substrate/frame/support/procedural/src/storage/genesis_config/builder_def.rs

@@ -88,7 +88,7 @@ impl BuilderDef {
 						}}
 					},
 					StorageLineTypeDef::Simple(_) => unreachable!(),
-					StorageLineTypeDef::Map(map) | StorageLineTypeDef::LinkedMap(map) => {
+					StorageLineTypeDef::Map(map) => {
 						let key = &map.key;
 						quote!{{
 							#data

substrate/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs

@@ -93,7 +93,7 @@ impl GenesisConfigDef {
 
 			let typ = match &line.storage_type {
 				StorageLineTypeDef::Simple(_) => (*value_type).clone(),
-				StorageLineTypeDef::Map(map) | StorageLineTypeDef::LinkedMap(map) => {
+				StorageLineTypeDef::Map(map) => {
 					let key = &map.key;
 					parse_quote!( Vec<(#key, #value_type)> )
 				},

substrate/frame/support/procedural/src/storage/getters.rs

@@ -40,7 +40,7 @@ pub fn impl_getters(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStrea
 					}
 				}
 			},
-			StorageLineTypeDef::Map(map) | StorageLineTypeDef::LinkedMap(map) => {
+			StorageLineTypeDef::Map(map) => {
 				let key = &map.key;
 				let value = &map.value;
 				quote!{

substrate/frame/support/procedural/src/storage/metadata.rs

@@ -41,20 +41,7 @@ fn storage_line_metadata_type(scrate: &TokenStream, line: &StorageLineDefExt) ->
 					hasher: #scrate::metadata::#hasher,
 					key: #scrate::metadata::DecodeDifferent::Encode(#key),
 					value: #scrate::metadata::DecodeDifferent::Encode(#value_type),
-					is_linked: false,
-				}
-			}
-		},
-		StorageLineTypeDef::LinkedMap(map) => {
-			let hasher = map.hasher.into_metadata();
-			let key = &map.key;
-			let key = clean_type_string(&quote!(#key).to_string());
-			quote!{
-				#scrate::metadata::StorageEntryType::Map {
-					hasher: #scrate::metadata::#hasher,
-					key: #scrate::metadata::DecodeDifferent::Encode(#key),
-					value: #scrate::metadata::DecodeDifferent::Encode(#value_type),
-					is_linked: true,
+					unused: false,
 				}
 			}
 		},

substrate/frame/support/procedural/src/storage/mod.rs

@@ -235,10 +235,6 @@ impl StorageLineDefExt {
 				ext::type_contains_ident(&map.key, &def.module_runtime_generic)
 					|| ext::type_contains_ident(&map.value, &def.module_runtime_generic)
 			}
-			StorageLineTypeDef::LinkedMap(map) => {
-				ext::type_contains_ident(&map.key, &def.module_runtime_generic)
-					|| ext::type_contains_ident(&map.value, &def.module_runtime_generic)
-			}
 			StorageLineTypeDef::DoubleMap(map) => {
 				ext::type_contains_ident(&map.key1, &def.module_runtime_generic)
 					|| ext::type_contains_ident(&map.key2, &def.module_runtime_generic)
@@ -249,7 +245,6 @@ impl StorageLineDefExt {
 		let query_type = match &storage_def.storage_type {
 			StorageLineTypeDef::Simple(value) => value.clone(),
 			StorageLineTypeDef::Map(map) => map.value.clone(),
-			StorageLineTypeDef::LinkedMap(map) => map.value.clone(),
 			StorageLineTypeDef::DoubleMap(map) => map.value.clone(),
 		};
 		let is_option = ext::extract_type_option(&query_type).is_some();
@@ -291,10 +286,6 @@ impl StorageLineDefExt {
 				let key = &map.key;
 				quote!( StorageMap<#key, #value_type> )
 			},
-			StorageLineTypeDef::LinkedMap(map) => {
-				let key = &map.key;
-				quote!( StorageLinkedMap<#key, #value_type> )
-			},
 			StorageLineTypeDef::DoubleMap(map) => {
 				let key1 = &map.key1;
 				let key2 = &map.key2;
@@ -336,7 +327,6 @@ impl StorageLineDefExt {
 
 pub enum StorageLineTypeDef {
 	Map(MapDef),
-	LinkedMap(MapDef),
 	DoubleMap(DoubleMapDef),
 	Simple(syn::Type),
 }
@@ -372,6 +362,7 @@ pub enum HasherKind {
 	Twox256,
 	Twox128,
 	Twox64Concat,
+	Identity,
 }
 
 impl HasherKind {
@@ -383,6 +374,7 @@ impl HasherKind {
 			HasherKind::Twox256 => quote!( Twox256 ),
 			HasherKind::Twox128 => quote!( Twox128 ),
 			HasherKind::Twox64Concat => quote!( Twox64Concat ),
+			HasherKind::Identity => quote!( Identity ),
 		}
 	}
 
@@ -394,6 +386,7 @@ impl HasherKind {
 			HasherKind::Twox256 => quote!( StorageHasher::Twox256 ),
 			HasherKind::Twox128 => quote!( StorageHasher::Twox128 ),
 			HasherKind::Twox64Concat => quote!( StorageHasher::Twox64Concat ),
+			HasherKind::Identity => quote!( StorageHasher::Identity ),
 		}
 	}
 }
@@ -420,7 +413,6 @@ pub fn decl_storage_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStr
 		use #scrate::{
 			StorageValue as _,
 			StorageMap as _,
-			StorageLinkedMap as _,
 			StorageDoubleMap as _,
 			StoragePrefixedMap as _,
 		};

substrate/frame/support/procedural/src/storage/parse.rs

@@ -27,15 +27,18 @@ mod keyword {
 	syn::custom_keyword!(build);
 	syn::custom_keyword!(get);
 	syn::custom_keyword!(map);
-	syn::custom_keyword!(linked_map);
 	syn::custom_keyword!(double_map);
-	syn::custom_keyword!(blake2_256);
-	syn::custom_keyword!(blake2_128);
+	syn::custom_keyword!(opaque_blake2_256);
+	syn::custom_keyword!(opaque_blake2_128);
 	syn::custom_keyword!(blake2_128_concat);
-	syn::custom_keyword!(twox_256);
-	syn::custom_keyword!(twox_128);
+	syn::custom_keyword!(opaque_twox_256);
+	syn::custom_keyword!(opaque_twox_128);
 	syn::custom_keyword!(twox_64_concat);
+	syn::custom_keyword!(identity);
 	syn::custom_keyword!(hasher);
+	syn::custom_keyword!(tainted);
+	syn::custom_keyword!(natural);
+	syn::custom_keyword!(prehashed);
 }
 
 /// Specific `Opt` to implement structure with optional parsing
@@ -194,7 +197,6 @@ impl_parse_for_opt!(DeclStorageBuild => keyword::build);
 #[derive(ToTokens, Debug)]
 enum DeclStorageType {
 	Map(DeclStorageMap),
-	LinkedMap(DeclStorageLinkedMap),
 	DoubleMap(DeclStorageDoubleMap),
 	Simple(syn::Type),
 }
@@ -203,8 +205,6 @@ impl syn::parse::Parse for DeclStorageType {
 	fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
 		if input.peek(keyword::map) {
 			Ok(Self::Map(input.parse()?))
-		} else if input.peek(keyword::linked_map) {
-			Ok(Self::LinkedMap(input.parse()?))
 		} else if input.peek(keyword::double_map) {
 			Ok(Self::DoubleMap(input.parse()?))
 		} else {
@@ -222,15 +222,6 @@ struct DeclStorageMap {
 	pub value: syn::Type,
 }
 
-#[derive(Parse, ToTokens, Debug)]
-struct DeclStorageLinkedMap {
-	pub map_keyword: keyword::linked_map,
-	pub hasher: Opt<SetHasher>,
-	pub key: syn::Type,
-	pub ass_keyword: Token![=>],
-	pub value: syn::Type,
-}
-
 #[derive(Parse, ToTokens, Debug)]
 struct DeclStorageDoubleMap {
 	pub map_keyword: keyword::double_map,
@@ -245,29 +236,38 @@ struct DeclStorageDoubleMap {
 
 #[derive(ToTokens, Debug)]
 enum Hasher {
-	Blake2_256(keyword::blake2_256),
-	Blake2_128(keyword::blake2_128),
+	Blake2_256(keyword::opaque_blake2_256),
+	Blake2_128(keyword::opaque_blake2_128),
 	Blake2_128Concat(keyword::blake2_128_concat),
-	Twox256(keyword::twox_256),
-	Twox128(keyword::twox_128),
+	Twox256(keyword::opaque_twox_256),
+	Twox128(keyword::opaque_twox_128),
 	Twox64Concat(keyword::twox_64_concat),
+	Identity(keyword::identity),
 }
 
 impl syn::parse::Parse for Hasher {
 	fn parse(input: syn::parse::ParseStream) -> syn::parse::Result<Self> {
 		let lookahead = input.lookahead1();
-		if lookahead.peek(keyword::blake2_256) {
+		if lookahead.peek(keyword::opaque_blake2_256) {
 			Ok(Self::Blake2_256(input.parse()?))
-		} else if lookahead.peek(keyword::blake2_128) {
+		} else if lookahead.peek(keyword::opaque_blake2_128) {
 			Ok(Self::Blake2_128(input.parse()?))
 		} else if lookahead.peek(keyword::blake2_128_concat) {
 			Ok(Self::Blake2_128Concat(input.parse()?))
-		} else if lookahead.peek(keyword::twox_256) {
+		} else if lookahead.peek(keyword::opaque_twox_256) {
 			Ok(Self::Twox256(input.parse()?))
-		} else if lookahead.peek(keyword::twox_128) {
+		} else if lookahead.peek(keyword::opaque_twox_128) {
 			Ok(Self::Twox128(input.parse()?))
 		} else if lookahead.peek(keyword::twox_64_concat) {
 			Ok(Self::Twox64Concat(input.parse()?))
+		} else if lookahead.peek(keyword::identity) {
+			Ok(Self::Identity(input.parse()?))
+		} else if lookahead.peek(keyword::tainted) {
+			Ok(Self::Blake2_128Concat(input.parse()?))
+		} else if lookahead.peek(keyword::natural) {
+			Ok(Self::Twox64Concat(input.parse()?))
+		} else if lookahead.peek(keyword::prehashed) {
+			Ok(Self::Identity(input.parse()?))
 		} else {
 			Err(lookahead.error())
 		}
@@ -313,6 +313,7 @@ impl From<Hasher> for super::HasherKind {
 			Hasher::Twox256(_) => super::HasherKind::Twox256,
 			Hasher::Twox128(_) => super::HasherKind::Twox128,
 			Hasher::Twox64Concat(_) => super::HasherKind::Twox64Concat,
+			Hasher::Identity(_) => super::HasherKind::Identity,
 		}
 	}
 }
@@ -464,7 +465,7 @@ fn parse_storage_line_defs(
 		let span = line.storage_type.span();
 		let no_hasher_error = || syn::Error::new(
 			span,
-			"Default hasher has been removed, use explicit hasher(blake2_256) instead."
+			"Default hasher has been removed, use explicit hasher(blake2_128_concat) instead."
 		);
 
 		let storage_type = match line.storage_type {
@@ -475,13 +476,6 @@ fn parse_storage_line_defs(
 					value: map.value,
 				}
 			),
-			DeclStorageType::LinkedMap(map) => super::StorageLineTypeDef::LinkedMap(
-				super::MapDef {
-					hasher: map.hasher.inner.ok_or_else(no_hasher_error)?.into(),
-					key: map.key,
-					value: map.value,
-				}
-			),
 			DeclStorageType::DoubleMap(map) => super::StorageLineTypeDef::DoubleMap(
 				super::DoubleMapDef {
 					hasher1: map.hasher1.inner.ok_or_else(no_hasher_error)?.into(),

substrate/frame/support/procedural/src/storage/storage_struct.rs

@@ -158,47 +158,6 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre
 					}
 				)
 			},
-			StorageLineTypeDef::LinkedMap(map) => {
-				let hasher = map.hasher.to_storage_hasher_struct();
-
-				let head_prefix_str = syn::LitStr::new(
-					&format!("HeadOf{}", line.name.to_string()),
-					line.name.span(),
-				);
-
-				quote!(
-					impl<#impl_trait> #scrate::#storage_generator_trait for #storage_struct
-					#optional_storage_where_clause
-					{
-						type Query = #query_type;
-						type KeyFormat = Self;
-
-						fn from_optional_value_to_query(v: Option<#value_type>) -> Self::Query {
-							#from_optional_value_to_query
-						}
-
-						fn from_query_to_optional_value(v: Self::Query) -> Option<#value_type> {
-							#from_query_to_optional_value
-						}
-					}
-
-					impl<#impl_trait> #scrate::storage::generator::LinkedMapKeyFormat for #storage_struct {
-						type Hasher = #scrate::#hasher;
-
-						fn module_prefix() -> &'static [u8] {
-							#instance_or_inherent::PREFIX.as_bytes()
-						}
-
-						fn storage_prefix() -> &'static [u8] {
-							#storage_name_str.as_bytes()
-						}
-
-						fn head_prefix() -> &'static [u8] {
-							#head_prefix_str.as_bytes()
-						}
-					}
-				)
-			},
 			StorageLineTypeDef::DoubleMap(map) => {
 				let hasher1 = map.hasher1.to_storage_hasher_struct();
 				let hasher2 = map.hasher2.to_storage_hasher_struct();