Implement StorageNMap (#8635)

* Implement StorageNMap * Change copyright date to 2021 * Rewrite keys to use impl_for_tuples instead of recursion * Implement prefix iteration on StorageNMap * Implement EncodeLike for key arguments * Rename KeyGenerator::Arg to KeyGenerator::KArg * Support StorageNMap in decl_storage and #[pallet::storage] macros * Use StorageNMap in assets pallet * Support migrate_keys in StorageNMap * Reduce line characters on select files * Refactor crate imports in decl_storage macros * Some more line char reductions and doc comment update * Update UI test expectations * Revert whitespace changes to untouched files * Generate Key struct instead of a 1-tuple when only 1 pair of key and hasher is provided * Revert formatting changes to unrelated files * Introduce KeyGeneratorInner * Add tests for StorageNMap in FRAMEv2 pallet macro * Small fixes to unit tests for StorageNMap * Bump runtime metadata version * Remove unused import * Update tests to use runtime metadata v13 * Introduce and use EncodeLikeTuple as a trait bound for KArg * Add some rustdocs * Revert usage of StorageNMap in assets pallet * Make use of ext::PunctuatedTrailing * Add rustdoc for final_hash * Fix StorageNMap proc macro expansions for single key cases * Create associated const in KeyGenerator for hasher metadata * Refactor code according to comments from Basti * Add module docs for generator/nmap.rs * Re-export storage::Key as NMapKey in pallet prelude * Seal the EncodeLikeTuple trait * Extract sealing code out of key.rs Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>
2026-04-28 07:27:55 +00:00 · 2021-05-14 02:44:29 -07:00
parent c6b1240e51
commit 033d8289f0
26 changed files with 3210 additions and 50 deletions
@@ -20,9 +20,16 @@
 use sp_core::storage::ChildInfo;
 use sp_std::prelude::*;
 use codec::{FullCodec, FullEncode, Encode, EncodeLike, Decode};
-use crate::hash::{Twox128, StorageHasher, ReversibleStorageHasher};
+use crate::{
+	hash::{Twox128, StorageHasher, ReversibleStorageHasher},
+	storage::types::{
+		EncodeLikeTuple, HasKeyPrefix, HasReversibleKeyPrefix, KeyGenerator,
+		ReversibleKeyGenerator, TupleToEncodedIter,
+	},
+};
 use sp_runtime::generic::{Digest, DigestItem};
 pub use sp_runtime::TransactionOutcome;
+pub use types::Key;

 pub mod unhashed;
 pub mod hashed;
@@ -359,6 +366,39 @@ pub trait IterableStorageDoubleMap<
 	fn translate<O: Decode, F: FnMut(K1, K2, O) -> Option<V>>(f: F);
 }

+/// A strongly-typed map with arbitrary number of keys in storage whose keys and values can be
+/// iterated over.
+pub trait IterableStorageNMap<K: ReversibleKeyGenerator, V: FullCodec>: StorageNMap<K, V> {
+	/// The type that iterates over all `(key1, (key2, (key3, ... (keyN, ()))), value)` tuples
+	type Iterator: Iterator<Item = (K::Key, V)>;
+
+	/// Enumerate all elements in the map with prefix key `kp` in no particular order. If you add or
+	/// remove values whose prefix is `kp` to the map while doing this, you'll get undefined
+	/// results.
+	fn iter_prefix<KP>(kp: KP) -> PrefixIterator<(<K as HasKeyPrefix<KP>>::Suffix, V)>
+	where K: HasReversibleKeyPrefix<KP>;
+
+	/// Remove all elements from the map with prefix key `kp` and iterate through them in no
+	/// particular order. If you add elements with prefix key `kp` to the map while doing this,
+	/// you'll get undefined results.
+	fn drain_prefix<KP>(kp: KP) -> PrefixIterator<(<K as HasKeyPrefix<KP>>::Suffix, V)>
+	where K: HasReversibleKeyPrefix<KP>;
+
+	/// Enumerate all elements in the map in no particular order. If you add or remove values to
+	/// the map while doing this, you'll get undefined results.
+	fn iter() -> Self::Iterator;
+
+	/// Remove all elements from the map and iterate through them in no particular order. If you
+	/// add elements to the map while doing this, you'll get undefined results.
+	fn drain() -> Self::Iterator;
+
+	/// Translate the values of all elements by a function `f`, in the map in no particular order.
+	/// By returning `None` from `f` for an element, you'll remove it from the map.
+	///
+	/// NOTE: If a value fail to decode because storage is corrupted then it is skipped.
+	fn translate<O: Decode, F: FnMut(K::Key, O) -> Option<V>>(f: F);
+}
+
 /// An implementation of a map with a two keys.
 ///
 /// It provides an important ability to efficiently remove all entries
@@ -510,6 +550,121 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 	>(key1: KeyArg1, key2: KeyArg2) -> Option<V>;
 }

+/// An implementation of a map with an arbitrary number of keys.
+///
+/// Details of implementation can be found at [`generator::StorageNMap`].
+pub trait StorageNMap<K: KeyGenerator, V: FullCodec> {
+	/// The type that get/take returns.
+	type Query;
+
+	/// Get the storage key used to fetch a value corresponding to a specific key.
+	fn hashed_key_for<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> Vec<u8>;
+
+	/// Does the value (explicitly) exist in storage?
+	fn contains_key<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> bool;
+
+	/// Load the value associated with the given key from the map.
+	fn get<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> Self::Query;
+
+	/// Try to get the value for the given key from the map.
+	///
+	/// Returns `Ok` if it exists, `Err` if not.
+	fn try_get<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> Result<V, ()>;
+
+	/// Swap the values of two keys.
+	fn swap<KOther, KArg1, KArg2>(key1: KArg1, key2: KArg2)
+	where
+		KOther: KeyGenerator,
+		KArg1: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		KArg2: EncodeLikeTuple<KOther::KArg> + TupleToEncodedIter;
+
+	/// Store a value to be associated with the given key from the map.
+	fn insert<KArg, VArg>(key: KArg, val: VArg)
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		VArg: EncodeLike<V>;
+
+	/// Remove the value under a key.
+	fn remove<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg);
+
+	/// Remove all values under the partial prefix key.
+	fn remove_prefix<KP>(partial_key: KP) where K: HasKeyPrefix<KP>;
+
+	/// Iterate over values that share the partial prefix key.
+	fn iter_prefix_values<KP>(partial_key: KP) -> PrefixIterator<V> where K: HasKeyPrefix<KP>;
+
+	/// Mutate the value under a key.
+	fn mutate<KArg, R, F>(key: KArg, f: F) -> R
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		F: FnOnce(&mut Self::Query) -> R;
+
+	/// Mutate the item, only if an `Ok` value is returned.
+	fn try_mutate<KArg, R, E, F>(key: KArg, f: F) -> Result<R, E>
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		F: FnOnce(&mut Self::Query) -> Result<R, E>;
+
+	/// Mutate the value under a key.
+	///
+	/// Deletes the item if mutated to a `None`.
+	fn mutate_exists<KArg, R, F>(key: KArg, f: F) -> R
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		F: FnOnce(&mut Option<V>) -> R;
+
+	/// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`.
+	fn try_mutate_exists<KArg, R, E, F>(key: KArg, f: F) -> Result<R, E>
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		F: FnOnce(&mut Option<V>) -> Result<R, E>;
+
+	/// Take the value under a key.
+	fn take<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> Self::Query;
+
+	/// Append the given items to the value in the storage.
+	///
+	/// `V` is required to implement `codec::EncodeAppend`.
+	///
+	/// # Warning
+	///
+	/// If the storage item is not encoded properly, the storage will be overwritten
+	/// and set to `[item]`. Any default value set for the storage item will be ignored
+	/// on overwrite.
+	fn append<Item, EncodeLikeItem, KArg>(key: KArg, item: EncodeLikeItem)
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter,
+		Item: Encode,
+		EncodeLikeItem: EncodeLike<Item>,
+		V: StorageAppend<Item>;
+
+	/// Read the length of the storage value without decoding the entire value under the
+	/// given `key`.
+	///
+	/// `V` is required to implement [`StorageDecodeLength`].
+	///
+	/// If the value does not exists or it fails to decode the length, `None` is returned.
+	/// Otherwise `Some(len)` is returned.
+	///
+	/// # Warning
+	///
+	/// `None` does not mean that `get()` does not return a value. The default value is completly
+	/// ignored by this function.
+	fn decode_len<KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter>(key: KArg) -> Option<usize>
+	where
+		V: StorageDecodeLength,
+	{
+		V::decode_len(&Self::hashed_key_for(key))
+	}
+
+	/// Migrate an item with the given `key` from defunct `hash_fns` to the current hashers.
+	///
+	/// If the key doesn't exist, then it's a no-op. If it does, then it returns its value.
+	fn migrate_keys<KArg>(key: KArg, hash_fns: K::HArg) -> Option<V>
+	where
+		KArg: EncodeLikeTuple<K::KArg> + TupleToEncodedIter;
+}
+
 /// Iterate over a prefix and decode raw_key and raw_value into `T`.
 ///
 /// If any decoding fails it skips it and continues to the next key.
@@ -806,7 +961,7 @@ pub trait StorageDecodeLength: private::Sealed + codec::DecodeLength {
 }

 /// Provides `Sealed` trait to prevent implementing trait `StorageAppend` & `StorageDecodeLength`
-/// outside of this crate.
+/// & `EncodeLikeTuple` outside of this crate.
 mod private {
 	use super::*;
 	use bounded_vec::BoundedVec;
@@ -818,6 +973,33 @@ mod private {
 	impl<T, S> Sealed for BoundedVec<T, S> {}
 	impl<K, V, S> Sealed for bounded_btree_map::BoundedBTreeMap<K, V, S> {}
 	impl<T, S> Sealed for bounded_btree_set::BoundedBTreeSet<T, S> {}
+
+	macro_rules! impl_sealed_for_tuple {
+		($($elem:ident),+) => {
+			paste::paste! {
+				impl<$($elem: Encode,)+> Sealed for ($($elem,)+) {}
+				impl<$($elem: Encode,)+> Sealed for &($($elem,)+) {}
+			}
+		};
+	}
+
+	impl_sealed_for_tuple!(A);
+	impl_sealed_for_tuple!(A, B);
+	impl_sealed_for_tuple!(A, B, C);
+	impl_sealed_for_tuple!(A, B, C, D);
+	impl_sealed_for_tuple!(A, B, C, D, E);
+	impl_sealed_for_tuple!(A, B, C, D, E, F);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q);
+	impl_sealed_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M, O, P, Q, R);
 }

 impl<T: Encode> StorageAppend<T> for Vec<T> {}