Implement a CountedStorageMap (#9125)

* initial impl

* expose in pallet_prelude

* temp test

* Apply suggestions from code review

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>
Co-authored-by: Xiliang Chen <xlchen1291@gmail.com>

* implement with macro help.

* test for macro generation

* add iterable functions, some test and fixes

* fix merge

* doc

* Update frame/support/src/storage/types/counted_map.rs

Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com>

* fix merge

* fmt

* fix spelling

* improve on removal

* fix partial storage info

* fmt

* add license

* suggested renames

* fix typo

* fix test

* fmt

* fix ui tests

* clearer doc

* better doc

* add metadata test

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>
Co-authored-by: Xiliang Chen <xlchen1291@gmail.com>
Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com>

substrate/frame/support/test/tests/pallet.rs

@@ -323,6 +323,12 @@ pub mod pallet {
 	pub type ConditionalNMap<T> =
 		StorageNMap<_, (storage::Key<Blake2_128Concat, u8>, storage::Key<Twox64Concat, u16>), u32>;
 
+	#[pallet::storage]
+	#[pallet::storage_prefix = "RenamedCountedMap"]
+	#[pallet::getter(fn counted_storage_map)]
+	pub type SomeCountedStorageMap<T> =
+		CountedStorageMap<Hasher = Twox64Concat, Key = u8, Value = u32>;
+
 	#[pallet::genesis_config]
 	#[derive(Default)]
 	pub struct GenesisConfig {
@@ -416,6 +422,7 @@ pub mod pallet {
 }
 
 // Test that a pallet with non generic event and generic genesis_config is correctly handled
+// and that a pallet without the attribute generate_storage_info is correctly handled.
 #[frame_support::pallet]
 pub mod pallet2 {
 	use super::{SomeAssociation1, SomeType1};
@@ -446,6 +453,10 @@ pub mod pallet2 {
 	#[pallet::storage]
 	pub type SomeValue<T: Config> = StorageValue<_, Vec<u32>>;
 
+	#[pallet::storage]
+	pub type SomeCountedStorageMap<T> =
+		CountedStorageMap<Hasher = Twox64Concat, Key = u8, Value = u32>;
+
 	#[pallet::event]
 	pub enum Event {
 		/// Something
@@ -899,6 +910,13 @@ fn storage_expand() {
 			pallet::ConditionalDoubleMap::<Runtime>::insert(1, 2, 3);
 			pallet::ConditionalNMap::<Runtime>::insert((1, 2), 3);
 		}
+
+		pallet::SomeCountedStorageMap::<Runtime>::insert(1, 2);
+		let mut k = [twox_128(b"Example"), twox_128(b"RenamedCountedMap")].concat();
+		k.extend(1u8.using_encoded(twox_64_concat));
+		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		let k = [twox_128(b"Example"), twox_128(b"CounterForRenamedCountedMap")].concat();
+		assert_eq!(unhashed::get::<u32>(&k), Some(1u32));
 	})
 }
 
@@ -1180,6 +1198,24 @@ fn metadata() {
 						default: vec![0],
 						docs: vec![],
 					},
+					StorageEntryMetadata {
+						name: "RenamedCountedMap",
+						modifier: StorageEntryModifier::Optional,
+						ty: StorageEntryType::Map {
+							hashers: vec![StorageHasher::Twox64Concat],
+							key: meta_type::<u8>(),
+							value: meta_type::<u32>(),
+						},
+						default: vec![0],
+						docs: vec![],
+					},
+					StorageEntryMetadata {
+						name: "CounterForRenamedCountedMap",
+						modifier: StorageEntryModifier::Default,
+						ty: StorageEntryType::Plain(meta_type::<u32>()),
+						default: vec![0, 0, 0, 0],
+						docs: vec!["Counter for the related counted storage map"],
+					},
 				],
 			}),
 			calls: Some(meta_type::<pallet::Call<Runtime>>().into()),
@@ -1370,6 +1406,24 @@ fn metadata() {
 						default: vec![0],
 						docs: vec![],
 					},
+					StorageEntryMetadata {
+						name: "RenamedCountedMap",
+						modifier: StorageEntryModifier::Optional,
+						ty: StorageEntryType::Map {
+							hashers: vec![StorageHasher::Twox64Concat],
+							key: meta_type::<u8>(),
+							value: meta_type::<u32>(),
+						},
+						default: vec![0],
+						docs: vec![],
+					},
+					StorageEntryMetadata {
+						name: "CounterForRenamedCountedMap",
+						modifier: StorageEntryModifier::Default,
+						ty: StorageEntryType::Plain(meta_type::<u32>()),
+						default: vec![0, 0, 0, 0],
+						docs: vec!["Counter for the related counted storage map"],
+					},
 				],
 			}),
 			calls: Some(meta_type::<pallet::Call<Runtime>>().into()),
@@ -1577,17 +1631,47 @@ fn test_storage_info() {
 					max_size: Some(7 + 16 + 8),
 				}
 			},
+			StorageInfo {
+				pallet_name: b"Example".to_vec(),
+				storage_name: b"RenamedCountedMap".to_vec(),
+				prefix: prefix(b"Example", b"RenamedCountedMap").to_vec(),
+				max_values: None,
+				max_size: Some(1 + 4 + 8),
+			},
+			StorageInfo {
+				pallet_name: b"Example".to_vec(),
+				storage_name: b"CounterForRenamedCountedMap".to_vec(),
+				prefix: prefix(b"Example", b"CounterForRenamedCountedMap").to_vec(),
+				max_values: Some(1),
+				max_size: Some(4),
+			},
 		],
 	);
 
 	assert_eq!(
 		Example2::storage_info(),
-		vec![StorageInfo {
-			pallet_name: b"Example2".to_vec(),
-			storage_name: b"SomeValue".to_vec(),
-			prefix: prefix(b"Example2", b"SomeValue").to_vec(),
-			max_values: Some(1),
-			max_size: None,
-		},],
+		vec![
+			StorageInfo {
+				pallet_name: b"Example2".to_vec(),
+				storage_name: b"SomeValue".to_vec(),
+				prefix: prefix(b"Example2", b"SomeValue").to_vec(),
+				max_values: Some(1),
+				max_size: None,
+			},
+			StorageInfo {
+				pallet_name: b"Example2".to_vec(),
+				storage_name: b"SomeCountedStorageMap".to_vec(),
+				prefix: prefix(b"Example2", b"SomeCountedStorageMap").to_vec(),
+				max_values: None,
+				max_size: None,
+			},
+			StorageInfo {
+				pallet_name: b"Example2".to_vec(),
+				storage_name: b"CounterForSomeCountedStorageMap".to_vec(),
+				prefix: prefix(b"Example2", b"CounterForSomeCountedStorageMap").to_vec(),
+				max_values: Some(1),
+				max_size: Some(4),
+			},
+		],
 	);
 }

substrate/frame/support/test/tests/pallet_ui/duplicate_storage_prefix.rs

@@ -1,6 +1,6 @@
 #[frame_support::pallet]
 mod pallet {
-	use frame_support::pallet_prelude::StorageValue;
+	use frame_support::pallet_prelude::*;
 
 	#[pallet::config]
 	pub trait Config: frame_system::Config {}
@@ -12,9 +12,15 @@ mod pallet {
 	#[pallet::storage]
 	type Foo<T> = StorageValue<_, u8>;
 
-    #[pallet::storage]
-    #[pallet::storage_prefix = "Foo"]
-    type NotFoo<T> = StorageValue<_, u16>;
+	#[pallet::storage]
+	#[pallet::storage_prefix = "Foo"]
+	type NotFoo<T> = StorageValue<_, u16>;
+
+	#[pallet::storage]
+	type CounterForBar<T> = StorageValue<_, u16>;
+
+	#[pallet::storage]
+	type Bar<T> = CountedStorageMap<_, Twox64Concat, u16, u16>;
 }
 
 fn main() {

substrate/frame/support/test/tests/pallet_ui/duplicate_storage_prefix.stderr

@@ -1,17 +1,47 @@
 error: Duplicate storage prefixes found for `Foo`
-  --> $DIR/duplicate_storage_prefix.rs:16:32
+  --> $DIR/duplicate_storage_prefix.rs:16:29
    |
 16 |     #[pallet::storage_prefix = "Foo"]
    |                                ^^^^^
 
+error: Duplicate storage prefixes found for `Foo`
+  --> $DIR/duplicate_storage_prefix.rs:13:7
+   |
+13 |     type Foo<T> = StorageValue<_, u8>;
+   |          ^^^
+
+error: Duplicate storage prefixes found for `CounterForBar`, used for counter associated to counted storage map
+  --> $DIR/duplicate_storage_prefix.rs:23:7
+   |
+23 |     type Bar<T> = CountedStorageMap<_, Twox64Concat, u16, u16>;
+   |          ^^^
+
+error: Duplicate storage prefixes found for `CounterForBar`
+  --> $DIR/duplicate_storage_prefix.rs:20:7
+   |
+20 |     type CounterForBar<T> = StorageValue<_, u16>;
+   |          ^^^^^^^^^^^^^
+
 error[E0412]: cannot find type `_GeneratedPrefixForStorageFoo` in this scope
   --> $DIR/duplicate_storage_prefix.rs:13:7
    |
 13 |     type Foo<T> = StorageValue<_, u8>;
    |          ^^^ not found in this scope
 
-error[E0121]: the type placeholder `_` is not allowed within types on item signatures
-  --> $DIR/duplicate_storage_prefix.rs:17:35
+error[E0412]: cannot find type `_GeneratedPrefixForStorageNotFoo` in this scope
+  --> $DIR/duplicate_storage_prefix.rs:17:7
    |
 17 |     type NotFoo<T> = StorageValue<_, u16>;
-   |                                   ^ not allowed in type signatures
+   |          ^^^^^^ not found in this scope
+
+error[E0412]: cannot find type `_GeneratedPrefixForStorageCounterForBar` in this scope
+  --> $DIR/duplicate_storage_prefix.rs:20:7
+   |
+20 |     type CounterForBar<T> = StorageValue<_, u16>;
+   |          ^^^^^^^^^^^^^ not found in this scope
+
+error[E0412]: cannot find type `_GeneratedPrefixForStorageBar` in this scope
+  --> $DIR/duplicate_storage_prefix.rs:23:7
+   |
+23 |     type Bar<T> = CountedStorageMap<_, Twox64Concat, u16, u16>;
+   |          ^^^ not found in this scope

substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr

@@ -5,8 +5,8 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied
    |               ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar`
    |
    = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12
@@ -16,8 +16,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
    |
    = note: required because of the requirements on the impl of `Decode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12
@@ -27,8 +27,8 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied
    |
    = note: required because of the requirements on the impl of `FullEncode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12
@@ -39,8 +39,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied
    = note: required because of the requirements on the impl of `Encode` for `Bar`
    = note: required because of the requirements on the impl of `FullEncode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
  --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:9:12

substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr

@@ -5,8 +5,8 @@ error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied
    |               ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar`
    |
    = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12
@@ -16,8 +16,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
    |
    = note: required because of the requirements on the impl of `Decode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12
@@ -27,8 +27,8 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied
    |
    = note: required because of the requirements on the impl of `FullEncode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied
   --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12
@@ -39,8 +39,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied
    = note: required because of the requirements on the impl of `Encode` for `Bar`
    = note: required because of the requirements on the impl of `FullEncode` for `Bar`
    = note: required because of the requirements on the impl of `FullCodec` for `Bar`
-   = note: required because of the requirements on the impl of `frame_support::storage::StorageEntryMetadata` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
-   = note: required by `NAME`
+   = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo<T>, Bar>`
+   = note: required by `build_metadata`
 
 error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied
  --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:9:12

substrate/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr

@@ -4,6 +4,6 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied
 10 |     #[pallet::generate_storage_info]
    |               ^^^^^^^^^^^^^^^^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar`
    |
-   = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `NMapKey<frame_support::Twox64Concat, Bar>`
-   = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo<T>, NMapKey<frame_support::Twox64Concat, Bar>, u32>`
+   = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `Key<frame_support::Twox64Concat, Bar>`
+   = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo<T>, Key<frame_support::Twox64Concat, Bar>, u32>`
    = note: required by `storage_info`