Implement StorageNMap (#8635)

* Implement StorageNMap

* Change copyright date to 2021

* Rewrite keys to use impl_for_tuples instead of recursion

* Implement prefix iteration on StorageNMap

* Implement EncodeLike for key arguments

* Rename KeyGenerator::Arg to KeyGenerator::KArg

* Support StorageNMap in decl_storage and #[pallet::storage] macros

* Use StorageNMap in assets pallet

* Support migrate_keys in StorageNMap

* Reduce line characters on select files

* Refactor crate imports in decl_storage macros

* Some more line char reductions and doc comment update

* Update UI test expectations

* Revert whitespace changes to untouched files

* Generate Key struct instead of a 1-tuple when only 1 pair of key and hasher is provided

* Revert formatting changes to unrelated files

* Introduce KeyGeneratorInner

* Add tests for StorageNMap in FRAMEv2 pallet macro

* Small fixes to unit tests for StorageNMap

* Bump runtime metadata version

* Remove unused import

* Update tests to use runtime metadata v13

* Introduce and use EncodeLikeTuple as a trait bound for KArg

* Add some rustdocs

* Revert usage of StorageNMap in assets pallet

* Make use of ext::PunctuatedTrailing

* Add rustdoc for final_hash

* Fix StorageNMap proc macro expansions for single key cases

* Create associated const in KeyGenerator for hasher metadata

* Refactor code according to comments from Basti

* Add module docs for generator/nmap.rs

* Re-export storage::Key as NMapKey in pallet prelude

* Seal the EncodeLikeTuple trait

* Extract sealing code out of key.rs

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/frame/support/test/tests/construct_runtime.rs

@@ -505,7 +505,7 @@ fn test_metadata() {
 			signed_extensions: vec![DecodeDifferent::Encode("UnitSignedExtension")],
 		},
 	};
-	pretty_assertions::assert_eq!(Runtime::metadata().1, RuntimeMetadata::V12(expected_metadata));
+	pretty_assertions::assert_eq!(Runtime::metadata().1, RuntimeMetadata::V13(expected_metadata));
 }
 
 #[test]

substrate/frame/support/test/tests/pallet.rs

@@ -217,6 +217,21 @@ pub mod pallet {
 	#[pallet::storage]
 	pub type DoubleMap2<T> = StorageDoubleMap<_, Twox64Concat, u16, Blake2_128Concat, u32, u64>;
 
+	#[pallet::storage]
+	#[pallet::getter(fn nmap)]
+	pub type NMap<T> = StorageNMap<_, storage::Key<Blake2_128Concat, u8>, u32>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn nmap2)]
+	pub type NMap2<T> = StorageNMap<
+		_,
+		(
+			NMapKey<Twox64Concat, u16>,
+			NMapKey<Blake2_128Concat, u32>,
+		),
+		u64,
+	>;
+
 	#[pallet::storage]
 	#[pallet::getter(fn conditional_value)]
 	#[cfg(feature = "conditional-storage")]
@@ -239,6 +254,18 @@ pub mod pallet {
 		u32,
 	>;
 
+	#[cfg(feature = "conditional-storage")]
+	#[pallet::storage]
+	#[pallet::getter(fn conditional_nmap)]
+	pub type ConditionalNMap<T> = StorageNMap<
+		_,
+		(
+			storage::Key<Blake2_128Concat, u8>,
+			storage::Key<Twox64Concat, u16>,
+		),
+		u32,
+	>;
+
 	#[pallet::genesis_config]
 	#[derive(Default)]
 	pub struct GenesisConfig {
@@ -578,11 +605,25 @@ fn storage_expand() {
 		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
 		assert_eq!(&k[..32], &<pallet::DoubleMap2<Runtime>>::final_prefix());
 
+		pallet::NMap::<Runtime>::insert((&1,), &3);
+		let mut k = [twox_128(b"Example"), twox_128(b"NMap")].concat();
+		k.extend(1u8.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<pallet::NMap<Runtime>>::final_prefix());
+
+		pallet::NMap2::<Runtime>::insert((&1, &2), &3);
+		let mut k = [twox_128(b"Example"), twox_128(b"NMap2")].concat();
+		k.extend(1u16.using_encoded(twox_64_concat));
+		k.extend(2u32.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
+		assert_eq!(&k[..32], &<pallet::NMap2<Runtime>>::final_prefix());
+
 		#[cfg(feature = "conditional-storage")]
 		{
 			pallet::ConditionalValue::<Runtime>::put(1);
 			pallet::ConditionalMap::<Runtime>::insert(1, 2);
 			pallet::ConditionalDoubleMap::<Runtime>::insert(1, 2, 3);
+			pallet::ConditionalNMap::<Runtime>::insert((1, 2), 3);
 		}
 	})
 }
@@ -708,6 +749,36 @@ fn metadata() {
 					default: DecodeDifferent::Decoded(vec![0]),
 					documentation: DecodeDifferent::Decoded(vec![]),
 				},
+				StorageEntryMetadata {
+					name: DecodeDifferent::Decoded("NMap".to_string()),
+					modifier: StorageEntryModifier::Optional,
+					ty: StorageEntryType::NMap {
+						keys: DecodeDifferent::Decoded(vec!["u8".to_string()]),
+						hashers: DecodeDifferent::Decoded(vec![
+							StorageHasher::Blake2_128Concat,
+						]),
+						value: DecodeDifferent::Decoded("u32".to_string()),
+					},
+					default: DecodeDifferent::Decoded(vec![0]),
+					documentation: DecodeDifferent::Decoded(vec![]),
+				},
+				StorageEntryMetadata {
+					name: DecodeDifferent::Decoded("NMap2".to_string()),
+					modifier: StorageEntryModifier::Optional,
+					ty: StorageEntryType::NMap {
+						keys: DecodeDifferent::Decoded(vec![
+							"u16".to_string(),
+							"u32".to_string(),
+						]),
+						hashers: DecodeDifferent::Decoded(vec![
+							StorageHasher::Twox64Concat,
+							StorageHasher::Blake2_128Concat,
+						]),
+						value: DecodeDifferent::Decoded("u64".to_string()),
+					},
+					default: DecodeDifferent::Decoded(vec![0]),
+					documentation: DecodeDifferent::Decoded(vec![]),
+				},
 				#[cfg(feature = "conditional-storage")] StorageEntryMetadata {
 					name: DecodeDifferent::Decoded("ConditionalValue".to_string()),
 					modifier: StorageEntryModifier::Optional,
@@ -740,6 +811,20 @@ fn metadata() {
 					default: DecodeDifferent::Decoded(vec![0]),
 					documentation: DecodeDifferent::Decoded(vec![]),
 				},
+				#[cfg(feature = "conditional-storage")] StorageEntryMetadata {
+					name: DecodeDifferent::Decoded("ConditionalNMap".to_string()),
+					modifier: StorageEntryModifier::Optional,
+					ty: StorageEntryType::NMap {
+						keys: DecodeDifferent::Decoded(vec!["u8".to_string(), "u16".to_string()]),
+						hashers: DecodeDifferent::Decoded(vec![
+							StorageHasher::Blake2_128Concat,
+							StorageHasher::Twox64Concat,
+						]),
+						value: DecodeDifferent::Decoded("u32".to_string()),
+					},
+					default: DecodeDifferent::Decoded(vec![0]),
+					documentation: DecodeDifferent::Decoded(vec![]),
+				},
 			]),
 		})),
 		calls: Some(DecodeDifferent::Decoded(vec![
@@ -857,7 +942,7 @@ fn metadata() {
 	};
 
 	let metadata = match Runtime::metadata().1 {
-		RuntimeMetadata::V12(metadata) => metadata,
+		RuntimeMetadata::V13(metadata) => metadata,
 		_ => panic!("metadata has been bump, test needs to be updated"),
 	};
 

substrate/frame/support/test/tests/pallet_compatibility.rs

@@ -266,7 +266,7 @@ mod test {
 	fn metadata() {
 		let metadata = Runtime::metadata();
 		let modules = match metadata.1 {
-			frame_metadata::RuntimeMetadata::V12(frame_metadata::RuntimeMetadataV12 {
+			frame_metadata::RuntimeMetadata::V13(frame_metadata::RuntimeMetadataV13 {
 				modules: frame_metadata::DecodeDifferent::Encode(m),
 				..
 			}) => m,

substrate/frame/support/test/tests/pallet_compatibility_instance.rs

@@ -281,7 +281,7 @@ mod test {
 	fn metadata() {
 		let metadata = Runtime::metadata();
 		let modules = match metadata.1 {
-			frame_metadata::RuntimeMetadata::V12(frame_metadata::RuntimeMetadataV12 {
+			frame_metadata::RuntimeMetadata::V13(frame_metadata::RuntimeMetadataV13 {
 				modules: frame_metadata::DecodeDifferent::Encode(m),
 				..
 			}) => m,

substrate/frame/support/test/tests/pallet_instance.rs

@@ -134,6 +134,21 @@ pub mod pallet {
 	pub type DoubleMap2<T, I = ()> =
 		StorageDoubleMap<_, Twox64Concat, u16, Blake2_128Concat, u32, u64>;
 
+	#[pallet::storage]
+	#[pallet::getter(fn nmap)]
+	pub type NMap<T, I = ()> = StorageNMap<_, storage::Key<Blake2_128Concat, u8>, u32>;
+
+	#[pallet::storage]
+	#[pallet::getter(fn nmap2)]
+	pub type NMap2<T, I = ()> = StorageNMap<
+		_,
+		(
+			storage::Key<Twox64Concat, u16>,
+			storage::Key<Blake2_128Concat, u32>,
+		),
+		u64,
+	>;
+
 	#[pallet::genesis_config]
 	#[derive(Default)]
 	pub struct GenesisConfig {
@@ -447,6 +462,19 @@ fn storage_expand() {
 		k.extend(2u32.using_encoded(blake2_128_concat));
 		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
 		assert_eq!(&k[..32], &<pallet::DoubleMap2<Runtime>>::final_prefix());
+
+		<pallet::NMap<Runtime>>::insert((&1,), &3);
+		let mut k = [twox_128(b"Example"), twox_128(b"NMap")].concat();
+		k.extend(1u8.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<pallet::NMap<Runtime>>::final_prefix());
+
+		<pallet::NMap2<Runtime>>::insert((&1, &2), &3);
+		let mut k = [twox_128(b"Example"), twox_128(b"NMap2")].concat();
+		k.extend(1u16.using_encoded(twox_64_concat));
+		k.extend(2u32.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
+		assert_eq!(&k[..32], &<pallet::NMap2<Runtime>>::final_prefix());
 	});
 
 	TestExternalities::default().execute_with(|| {
@@ -479,6 +507,19 @@ fn storage_expand() {
 		k.extend(2u32.using_encoded(blake2_128_concat));
 		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
 		assert_eq!(&k[..32], &<pallet::DoubleMap2<Runtime, pallet::Instance1>>::final_prefix());
+
+		<pallet::NMap<Runtime, pallet::Instance1>>::insert((&1,), &3);
+		let mut k = [twox_128(b"Instance1Example"), twox_128(b"NMap")].concat();
+		k.extend(1u8.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<pallet::NMap<Runtime, pallet::Instance1>>::final_prefix());
+
+		<pallet::NMap2<Runtime, pallet::Instance1>>::insert((&1, &2), &3);
+		let mut k = [twox_128(b"Instance1Example"), twox_128(b"NMap2")].concat();
+		k.extend(1u16.using_encoded(twox_64_concat));
+		k.extend(2u32.using_encoded(blake2_128_concat));
+		assert_eq!(unhashed::get::<u64>(&k), Some(3u64));
+		assert_eq!(&k[..32], &<pallet::NMap2<Runtime, pallet::Instance1>>::final_prefix());
 	});
 }
 
@@ -617,6 +658,36 @@ fn metadata() {
 					default: DecodeDifferent::Decoded(vec![0]),
 					documentation: DecodeDifferent::Decoded(vec![]),
 				},
+				StorageEntryMetadata {
+					name: DecodeDifferent::Decoded("NMap".to_string()),
+					modifier: StorageEntryModifier::Optional,
+					ty: StorageEntryType::NMap {
+						keys: DecodeDifferent::Decoded(vec!["u8".to_string()]),
+						hashers: DecodeDifferent::Decoded(vec![
+							StorageHasher::Blake2_128Concat,
+						]),
+						value: DecodeDifferent::Decoded("u32".to_string()),
+					},
+					default: DecodeDifferent::Decoded(vec![0]),
+					documentation: DecodeDifferent::Decoded(vec![]),
+				},
+				StorageEntryMetadata {
+					name: DecodeDifferent::Decoded("NMap2".to_string()),
+					modifier: StorageEntryModifier::Optional,
+					ty: StorageEntryType::NMap {
+						keys: DecodeDifferent::Decoded(vec![
+							"u16".to_string(),
+							"u32".to_string(),
+						]),
+						hashers: DecodeDifferent::Decoded(vec![
+							StorageHasher::Twox64Concat,
+							StorageHasher::Blake2_128Concat,
+						]),
+						value: DecodeDifferent::Decoded("u64".to_string()),
+					},
+					default: DecodeDifferent::Decoded(vec![0]),
+					documentation: DecodeDifferent::Decoded(vec![]),
+				},
 			]),
 		})),
 		calls: Some(DecodeDifferent::Decoded(vec![
@@ -696,7 +767,7 @@ fn metadata() {
 
 
 	let metadata = match Runtime::metadata().1 {
-		RuntimeMetadata::V12(metadata) => metadata,
+		RuntimeMetadata::V13(metadata) => metadata,
 		_ => panic!("metadata has been bump, test needs to be updated"),
 	};
 

substrate/frame/support/test/tests/pallet_ui/storage_not_storage_type.stderr

@@ -1,4 +1,4 @@
-error: Invalid pallet::storage, expected ident: `StorageValue` or `StorageMap` or `StorageDoubleMap` in order to expand metadata, found `u8`
+error: Invalid pallet::storage, expected ident: `StorageValue` or `StorageMap` or `StorageDoubleMap` or `StorageNMap` in order to expand metadata, found `u8`
   --> $DIR/storage_not_storage_type.rs:19:16
    |
 19 |     type Foo<T> = u8;