Introduce prefixed storage with enumeration (#4185)

* Introduce storage_next allowing iteration.  (without childtries)

* Implement prefixed storage

* impl cache in client_storage_cache (needs test)

* switch overlay change to btreemap

* Revert "impl cache in client_storage_cache"

This reverts commit c91a4848916eba87184b3dc4722cea81aec9339d.

the storage cache cannot be used this way

* Revert "Implement prefixed storage"

This reverts commit 4931088126a427082d7310ed7e83b8eea966bc20.

* Impl StoragePrefixedMap for all map storages

* remove comment

* Move all overlays to BTreeMap

* btreemap iteration improvment

* impl for child tries

* impl tests for childs

* fix

* remove cache comment

* Fix grumble

substrate/frame/support/procedural/src/lib.rs

@@ -69,6 +69,7 @@ use proc_macro::TokenStream;
 /// * Map: `Foo: map hasher($hash) type => type`: Implements the
 ///   [`StorageMap`](../frame_support/storage/trait.StorageMap.html) trait using the
 ///   [`StorageMap generator`](../frame_support/storage/generator/trait.StorageMap.html).
+///   And [`StoragePrefixedMap`](../frame_support/storage/trait.StoragePrefixedMap.html).
 ///
 ///   `$hash` representing a choice of hashing algorithms available in the
 ///   [`Hashable`](../frame_support/trait.Hashable.html) trait.
@@ -89,6 +90,7 @@ use proc_macro::TokenStream;
 /// * Linked map: `Foo: linked_map hasher($hash) type => type`: Implements the
 ///   [`StorageLinkedMap`](../frame_support/storage/trait.StorageLinkedMap.html) trait using the
 ///   [`StorageLinkedMap generator`](../frame_support/storage/generator/trait.StorageLinkedMap.html).
+///   And [`StoragePrefixedMap`](../frame_support/storage/trait.StoragePrefixedMap.html).
 ///
 ///   `$hash` representing a choice of hashing algorithms available in the
 ///   [`Hashable`](../frame_support/trait.Hashable.html) trait.
@@ -118,6 +120,7 @@ use proc_macro::TokenStream;
 /// * Double map: `Foo: double_map hasher($hash1) u32, $hash2(u32) => u32`: Implements the
 ///   [`StorageDoubleMap`](../frame_support/storage/trait.StorageDoubleMap.html) trait using the
 ///   [`StorageDoubleMap generator`](../frame_support/storage/generator/trait.StorageDoubleMap.html).
+///   And [`StoragePrefixedMap`](../frame_support/storage/trait.StoragePrefixedMap.html).
 ///
 ///   `$hash1` and `$hash2` representing choices of hashing algorithms available in the
 ///   [`Hashable`](../frame_support/trait.Hashable.html) trait. They must be choosen with care, see

substrate/frame/support/procedural/src/storage/mod.rs

@@ -418,7 +418,8 @@ pub fn decl_storage_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStr
 			StorageValue as _,
 			StorageMap as _,
 			StorageLinkedMap as _,
-			StorageDoubleMap as _
+			StorageDoubleMap as _,
+			StoragePrefixedMap as _,
 		};
 
 		#scrate_decl

substrate/frame/support/procedural/src/storage/storage_struct.rs

@@ -122,6 +122,18 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre
 			StorageLineTypeDef::Map(map) => {
 				let hasher = map.hasher.to_storage_hasher_struct();
 				quote!(
+					impl<#impl_trait> #scrate::storage::StoragePrefixedMap<#value_type>
+						for #storage_struct #optional_storage_where_clause
+					{
+						fn module_prefix() -> &'static [u8] {
+							#instance_or_inherent::PREFIX.as_bytes()
+						}
+
+						fn storage_prefix() -> &'static [u8] {
+							#storage_name_str.as_bytes()
+						}
+					}
+
 					impl<#impl_trait> #scrate::#storage_generator_trait for #storage_struct
 					#optional_storage_where_clause
 					{
@@ -155,6 +167,18 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre
 				);
 
 				quote!(
+					impl<#impl_trait> #scrate::storage::StoragePrefixedMap<#value_type>
+						for #storage_struct #optional_storage_where_clause
+					{
+						fn module_prefix() -> &'static [u8] {
+							#instance_or_inherent::PREFIX.as_bytes()
+						}
+
+						fn storage_prefix() -> &'static [u8] {
+							#storage_name_str.as_bytes()
+						}
+					}
+
 					impl<#impl_trait> #scrate::#storage_generator_trait for #storage_struct
 					#optional_storage_where_clause
 					{
@@ -191,6 +215,18 @@ pub fn decl_and_impl(scrate: &TokenStream, def: &DeclStorageDefExt) -> TokenStre
 				let hasher1 = map.hasher1.to_storage_hasher_struct();
 				let hasher2 = map.hasher2.to_storage_hasher_struct();
 				quote!(
+					impl<#impl_trait> #scrate::storage::StoragePrefixedMap<#value_type>
+						for #storage_struct #optional_storage_where_clause
+					{
+						fn module_prefix() -> &'static [u8] {
+							#instance_or_inherent::PREFIX.as_bytes()
+						}
+
+						fn storage_prefix() -> &'static [u8] {
+							#storage_name_str.as_bytes()
+						}
+					}
+
 					impl<#impl_trait> #scrate::#storage_generator_trait for #storage_struct
 					#optional_storage_where_clause
 					{

substrate/frame/support/src/lib.rs

@@ -67,7 +67,9 @@ pub mod traits;
 pub mod weights;
 
 pub use self::hash::{Twox256, Twox128, Blake2_256, Blake2_128, Twox64Concat, Hashable};
-pub use self::storage::{StorageValue, StorageMap, StorageLinkedMap, StorageDoubleMap};
+pub use self::storage::{
+	StorageValue, StorageMap, StorageLinkedMap, StorageDoubleMap, StoragePrefixedMap
+};
 pub use self::dispatch::{Parameter, Callable, IsSubType};
 pub use sp_runtime::{self, ConsensusEngineId, print, traits::Printable};
 

substrate/frame/support/src/storage/mod.rs

@@ -16,9 +16,9 @@
 
 //! Stuff to do with the runtime's storage.
 
-use rstd::prelude::*;
+use rstd::{prelude::*, marker::PhantomData};
 use codec::{FullCodec, FullEncode, Encode, EncodeAppend, EncodeLike, Decode};
-use crate::traits::Len;
+use crate::{traits::Len, hash::{Twox128, StorageHasher}};
 
 pub mod unhashed;
 pub mod hashed;
@@ -352,3 +352,126 @@ pub trait StorageDoubleMap<K1: FullEncode, K2: FullEncode, V: FullCodec> {
 			KArg2: EncodeLike<K2>,
 			V: codec::DecodeLength + Len;
 }
+
+/// Iterator for prefixed map.
+pub struct PrefixIterator<Value> {
+	prefix: Vec<u8>,
+	previous_key: Vec<u8>,
+	phantom_data: PhantomData<Value>,
+}
+
+impl<Value: Decode> Iterator for PrefixIterator<Value> {
+	type Item = Value;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		match runtime_io::storage::next_key(&self.previous_key) {
+			Some(next_key) if next_key.starts_with(&self.prefix[..]) => {
+				let value = unhashed::get(&next_key);
+
+				if value.is_none() {
+					runtime_print!(
+						"ERROR: returned next_key has no value:\nkey is {:?}\nnext_key is {:?}",
+						&self.previous_key, &next_key,
+					);
+				}
+
+				self.previous_key = next_key;
+
+				value
+			},
+			_ => None,
+		}
+	}
+}
+
+/// Trait for maps that store all its value after a unique prefix.
+///
+/// By default the final prefix is:
+/// ```nocompile
+/// Twox128(module_prefix) ++ Twox128(storage_prefix)
+/// ```
+pub trait StoragePrefixedMap<Value: FullCodec> {
+
+	/// Module prefix. Used for generating final key.
+	fn module_prefix() -> &'static [u8];
+
+	/// Storage prefix. Used for generating final key.
+	fn storage_prefix() -> &'static [u8];
+
+	fn final_prefix() -> [u8; 32] {
+		let mut final_key = [0u8; 32];
+		final_key[0..16].copy_from_slice(&Twox128::hash(Self::module_prefix()));
+		final_key[16..32].copy_from_slice(&Twox128::hash(Self::storage_prefix()));
+		final_key
+	}
+
+	fn remove_all() {
+		runtime_io::storage::clear_prefix(&Self::final_prefix())
+	}
+
+	fn iter() -> PrefixIterator<Value> {
+		let prefix = Self::final_prefix();
+		PrefixIterator {
+			prefix: prefix.to_vec(),
+			previous_key: prefix.to_vec(),
+			phantom_data: Default::default(),
+		}
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use primitives::hashing::twox_128;
+	use runtime_io::TestExternalities;
+	use crate::storage::{unhashed, StoragePrefixedMap};
+
+	#[test]
+	fn prefixed_map_works() {
+		TestExternalities::default().execute_with(|| {
+			struct MyStorage;
+			impl StoragePrefixedMap<u64> for MyStorage {
+				fn module_prefix() -> &'static [u8] {
+					b"MyModule"
+				}
+
+				fn storage_prefix() -> &'static [u8] {
+					b"MyStorage"
+				}
+			}
+
+			let key_before = {
+				let mut k = MyStorage::final_prefix();
+				let last = k.iter_mut().last().unwrap();
+				*last = last.checked_sub(1).unwrap();
+				k
+			};
+			let key_after = {
+				let mut k = MyStorage::final_prefix();
+				let last = k.iter_mut().last().unwrap();
+				*last = last.checked_add(1).unwrap();
+				k
+			};
+
+			unhashed::put(&key_before[..], &32u64);
+			unhashed::put(&key_after[..], &33u64);
+
+			let k = [twox_128(b"MyModule"), twox_128(b"MyStorage")].concat();
+			assert_eq!(MyStorage::final_prefix().to_vec(), k);
+
+			assert_eq!(MyStorage::iter().collect::<Vec<_>>(), vec![]);
+
+			unhashed::put(&[&k[..], &vec![1][..]].concat(), &1u64);
+			unhashed::put(&[&k[..], &vec![1, 1][..]].concat(), &2u64);
+			unhashed::put(&[&k[..], &vec![8][..]].concat(), &3u64);
+			unhashed::put(&[&k[..], &vec![10][..]].concat(), &4u64);
+
+			assert_eq!(MyStorage::iter().collect::<Vec<_>>(), vec![1, 2, 3, 4]);
+
+			MyStorage::remove_all();
+
+			assert_eq!(MyStorage::iter().collect::<Vec<_>>(), vec![]);
+			assert_eq!(unhashed::get(&key_before[..]), Some(32u64));
+			assert_eq!(unhashed::get(&key_after[..]), Some(33u64));
+		});
+	}
+}

substrate/frame/support/test/tests/final_keys.rs

@@ -16,7 +16,7 @@
 
 use support::storage::unhashed;
 use codec::Encode;
-use support::{StorageDoubleMap, StorageLinkedMap, StorageMap, StorageValue};
+use support::{StorageDoubleMap, StorageLinkedMap, StorageMap, StorageValue, StoragePrefixedMap};
 use runtime_io::{TestExternalities, hashing::{twox_128, blake2_128, blake2_256}};
 
 mod no_instance {
@@ -96,11 +96,13 @@ fn final_keys_no_instance() {
 		let mut k = [twox_128(b"FinalKeysNone"), twox_128(b"Map")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<no_instance::Map>::final_prefix());
 
 		no_instance::Map2::insert(1, 2);
 		let mut k = [twox_128(b"FinalKeysNone"), twox_128(b"Map2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<no_instance::Map2>::final_prefix());
 
 		let head = [twox_128(b"FinalKeysNone"), twox_128(b"HeadOfLinkedMap")].concat();
 		assert_eq!(unhashed::get::<u32>(&head), None);
@@ -110,23 +112,27 @@ fn final_keys_no_instance() {
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
 		assert_eq!(unhashed::get::<u32>(&head), Some(1u32));
+		assert_eq!(&k[..32], &<no_instance::LinkedMap>::final_prefix());
 
 		no_instance::LinkedMap2::insert(1, 2);
 		let mut k = [twox_128(b"FinalKeysNone"), twox_128(b"LinkedMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<no_instance::LinkedMap2>::final_prefix());
 
 		no_instance::DoubleMap::insert(&1, &2, &3);
 		let mut k = [twox_128(b"FinalKeysNone"), twox_128(b"DoubleMap")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		k.extend(2u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<no_instance::DoubleMap>::final_prefix());
 
 		no_instance::DoubleMap2::insert(&1, &2, &3);
 		let mut k = [twox_128(b"FinalKeysNone"), twox_128(b"DoubleMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		k.extend(2u32.using_encoded(blake2_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<no_instance::DoubleMap2>::final_prefix());
 	});
 }
 
@@ -141,11 +147,13 @@ fn final_keys_default_instance() {
 		let mut k = [twox_128(b"FinalKeysSome"), twox_128(b"Map")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::Map<instance::DefaultInstance>>::final_prefix());
 
 		<instance::Map2<instance::DefaultInstance>>::insert(1, 2);
 		let mut k = [twox_128(b"FinalKeysSome"), twox_128(b"Map2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::Map2<instance::DefaultInstance>>::final_prefix());
 
 		let head = [twox_128(b"FinalKeysSome"), twox_128(b"HeadOfLinkedMap")].concat();
 		assert_eq!(unhashed::get::<u32>(&head), None);
@@ -155,23 +163,27 @@ fn final_keys_default_instance() {
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
 		assert_eq!(unhashed::get::<u32>(&head), Some(1u32));
+		assert_eq!(&k[..32], &<instance::LinkedMap<instance::DefaultInstance>>::final_prefix());
 
 		<instance::LinkedMap2<instance::DefaultInstance>>::insert(1, 2);
 		let mut k = [twox_128(b"FinalKeysSome"), twox_128(b"LinkedMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::LinkedMap2<instance::DefaultInstance>>::final_prefix());
 
 		<instance::DoubleMap<instance::DefaultInstance>>::insert(&1, &2, &3);
 		let mut k = [twox_128(b"FinalKeysSome"), twox_128(b"DoubleMap")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		k.extend(2u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<instance::DoubleMap<instance::DefaultInstance>>::final_prefix());
 
 		<instance::DoubleMap2<instance::DefaultInstance>>::insert(&1, &2, &3);
 		let mut k = [twox_128(b"FinalKeysSome"), twox_128(b"DoubleMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		k.extend(2u32.using_encoded(blake2_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<instance::DoubleMap2<instance::DefaultInstance>>::final_prefix());
 	});
 }
 
@@ -186,11 +198,13 @@ fn final_keys_instance_2() {
 		let mut k = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"Map")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::Map<instance::Instance2>>::final_prefix());
 
 		<instance::Map2<instance::Instance2>>::insert(1, 2);
 		let mut k = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"Map2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::Map2<instance::Instance2>>::final_prefix());
 
 		let head = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"HeadOfLinkedMap")].concat();
 		assert_eq!(unhashed::get::<u32>(&head), None);
@@ -200,22 +214,26 @@ fn final_keys_instance_2() {
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
 		assert_eq!(unhashed::get::<u32>(&head), Some(1u32));
+		assert_eq!(&k[..32], &<instance::LinkedMap<instance::Instance2>>::final_prefix());
 
 		<instance::LinkedMap2<instance::Instance2>>::insert(1, 2);
 		let mut k = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"LinkedMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(2u32));
+		assert_eq!(&k[..32], &<instance::LinkedMap2<instance::Instance2>>::final_prefix());
 
 		<instance::DoubleMap<instance::Instance2>>::insert(&1, &2, &3);
 		let mut k = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"DoubleMap")].concat();
 		k.extend(1u32.using_encoded(blake2_256).to_vec());
 		k.extend(2u32.using_encoded(blake2_256).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<instance::DoubleMap<instance::Instance2>>::final_prefix());
 
 		<instance::DoubleMap2<instance::Instance2>>::insert(&1, &2, &3);
 		let mut k = [twox_128(b"Instance2FinalKeysSome"), twox_128(b"DoubleMap2")].concat();
 		k.extend(1u32.using_encoded(twox_128).to_vec());
 		k.extend(2u32.using_encoded(blake2_128).to_vec());
 		assert_eq!(unhashed::get::<u32>(&k), Some(3u32));
+		assert_eq!(&k[..32], &<instance::DoubleMap2<instance::Instance2>>::final_prefix());
 	});
 }

substrate/frame/support/test/tests/instance.rs

@@ -300,7 +300,7 @@ fn new_test_ext() -> runtime_io::TestExternalities {
 
 #[test]
 fn storage_instance_independance() {
-	let mut storage = (std::collections::HashMap::new(), std::collections::HashMap::new());
+	let mut storage = Default::default();
 	state_machine::BasicExternalities::execute_with_storage(&mut storage, || {
 		module2::Value::<Runtime>::put(0);
 		module2::Value::<Runtime, module2::Instance1>::put(0);