EnumerableStorageMap (#1763)

* Refactor decl_storage a bit to allow easier impl of linked map.

* A bunch of refactorings for storage generation.

- Rename StorageMap and ChildrenStorageMap to avoid confusion with generator::StorageMap.
- Separate implementation from the procedural macro code to clean it up.
- Make sure that genesis is initialised using the `StorageValue/StorageMap`
  generated implementations instead of going RAW.

* WiP: Writing test.

* Basic implementation.

* Implement enumeration.

* Fix non-std issues.

* fix warning

* Fix test-client.

* Address review grumbles - part 1

* Avoid cloning the key, relax Storage requirements.

* Rebuild runtime.

* Remove dangling todo.

substrate/srml/support/src/lib.rs

@@ -54,7 +54,7 @@ pub mod inherent;
 mod double_map;
 pub mod traits;
 
-pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap};
+pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap, EnumerableStorageMap};
 pub use self::hashable::Hashable;
 pub use self::dispatch::{Parameter, Dispatchable, Callable, IsSubType};
 pub use self::double_map::StorageDoubleMap;
@@ -132,3 +132,125 @@ macro_rules! for_each_tuple {
 		for_each_tuple! { @IMPL $m !! $($t,)* }
 	}
 }
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use runtime_io::{with_externalities, Blake2Hasher};
+	use runtime_primitives::BuildStorage;
+
+	pub trait Trait {
+		type BlockNumber;
+		type Origin;
+	}
+
+	mod module {
+		#![allow(dead_code)]
+
+		use super::Trait;
+
+		decl_module! {
+			pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+
+			}
+		}
+	}
+	use self::module::Module;
+
+	decl_storage! {
+		trait Store for Module<T: Trait> as Example {
+			pub Data get(data) build(|_| vec![(15u32, 42u64)]): linked_map u32 => u64;
+		}
+	}
+
+	struct Test;
+	impl Trait for Test {
+		type BlockNumber = u32;
+		type Origin = u32;
+	}
+
+	fn new_test_ext() -> runtime_io::TestExternalities<Blake2Hasher> {
+		GenesisConfig::<Test>::default().build_storage().unwrap().0.into()
+	}
+
+	type Map = Data<Test>;
+
+	#[test]
+	fn basic_insert_remove_should_work() {
+		with_externalities(&mut new_test_ext(), || {
+			// initialised during genesis
+			assert_eq!(Map::get(&15u32), 42u64);
+
+			// get / insert / take
+			let key = 17u32;
+			assert_eq!(Map::get(&key), 0u64);
+			Map::insert(key, 4u64);
+			assert_eq!(Map::get(&key), 4u64);
+			assert_eq!(Map::take(&key), 4u64);
+			assert_eq!(Map::get(&key), 0u64);
+
+			// mutate
+			Map::mutate(&key, |val| {
+				*val = 15;
+			});
+			assert_eq!(Map::get(&key), 15u64);
+
+			// remove
+			Map::remove(&key);
+			assert_eq!(Map::get(&key), 0u64);
+		});
+	}
+
+	#[test]
+	fn enumeration_and_head_should_work() {
+		with_externalities(&mut new_test_ext(), || {
+			assert_eq!(Map::head(), Some(15));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(15, 42)]);
+			// insert / remove
+			let key = 17u32;
+			Map::insert(key, 4u64);
+			assert_eq!(Map::head(), Some(key));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 4), (15, 42)]);
+			assert_eq!(Map::take(&15), 42u64);
+			assert_eq!(Map::take(&key), 4u64);
+			assert_eq!(Map::head(), None);
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![]);
+
+			// Add couple of more elements
+			Map::insert(key, 42u64);
+			assert_eq!(Map::head(), Some(key));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 42)]);
+			Map::insert(key + 1, 43u64);
+			assert_eq!(Map::head(), Some(key + 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key + 1, 43), (key, 42)]);
+
+			// mutate
+			let key = key + 2;
+			Map::mutate(&key, |val| {
+				*val = 15;
+			});
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 15), (key - 1, 43), (key - 2, 42)]);
+			assert_eq!(Map::head(), Some(key));
+			Map::mutate(&key, |val| {
+				*val = 17;
+			});
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 17), (key - 1, 43), (key - 2, 42)]);
+
+			// remove first
+			Map::remove(&key);
+			assert_eq!(Map::head(), Some(key - 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key - 1, 43), (key - 2, 42)]);
+
+			// remove last from the list
+			Map::remove(&(key - 2));
+			assert_eq!(Map::head(), Some(key - 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key - 1, 43)]);
+
+			// remove the last element
+			Map::remove(&(key - 1));
+			assert_eq!(Map::head(), None);
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![]);
+		});
+	}
+
+}

substrate/srml/support/src/storage/generator.rs

@@ -52,6 +52,8 @@ use crate::rstd::vec::Vec;
 pub use crate::rstd::borrow::Borrow;
 #[doc(hidden)]
 pub use crate::rstd::marker::PhantomData;
+#[doc(hidden)]
+pub use crate::rstd::boxed::Box;
 
 pub use srml_metadata::{
 	DecodeDifferent, StorageMetadata, StorageFunctionMetadata,
@@ -65,34 +67,55 @@ pub trait Storage {
 	fn exists(&self, key: &[u8]) -> bool;
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
-	fn get<T: codec::Codec>(&self, key: &[u8]) -> Option<T>;
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T>;
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect. Will panic if
 	/// it's not there.
-	fn require<T: codec::Codec>(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") }
+	fn require<T: codec::Decode>(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") }
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect. The type's
 	/// default is returned if it's not there.
-	fn get_or_default<T: codec::Codec + Default>(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() }
+	fn get_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() }
 
 	/// Put a value in under a key.
-	fn put<T: codec::Codec>(&self, key: &[u8], val: &T);
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T);
 
 	/// Remove the bytes of a key from storage.
 	fn kill(&self, key: &[u8]);
 
 	/// Take a value from storage, deleting it after reading.
-	fn take<T: codec::Codec>(&self, key: &[u8]) -> Option<T> {
+	fn take<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
 		let value = self.get(key);
 		self.kill(key);
 		value
 	}
 
 	/// Take a value from storage, deleting it after reading.
-	fn take_or_panic<T: codec::Codec>(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") }
+	fn take_or_panic<T: codec::Decode>(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") }
 
 	/// Take a value from storage, deleting it after reading.
-	fn take_or_default<T: codec::Codec + Default>(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() }
+	fn take_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() }
+}
+
+// We use a construct like this during when genesis storage is being built.
+#[cfg(feature = "std")]
+impl<S: sr_primitives::BuildStorage> Storage for (crate::rstd::cell::RefCell<&mut sr_primitives::StorageOverlay>, PhantomData<S>) {
+	fn exists(&self, key: &[u8]) -> bool {
+		self.0.borrow().contains_key(S::hash(key).as_ref())
+	}
+
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
+		self.0.borrow().get(S::hash(key).as_ref())
+			.map(|x| codec::Decode::decode(&mut x.as_slice()).expect("Unable to decode expected type."))
+	}
+
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T) {
+		self.0.borrow_mut().insert(S::hash(key).to_vec(), codec::Encode::encode(val));
+	}
+
+	fn kill(&self, key: &[u8]) {
+		self.0.borrow_mut().remove(S::hash(key).as_ref());
+	}
 }
 
 /// A strongly-typed value kept in storage.
@@ -194,6 +217,15 @@ pub trait StorageMap<K: codec::Codec, V: codec::Codec> {
 	fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: Storage>(key: &K, f: F, storage: &S) -> R;
 }
 
+/// A `StorageMap` with enumerable entries.
+pub trait EnumerableStorageMap<K: codec::Codec, V: codec::Codec>: StorageMap<K, V> {
+	/// Return current head element.
+	fn head<S: Storage>(storage: &S) -> Option<K>;
+
+	/// Enumerate all elements in the map.
+	fn enumerate<'a, S: Storage>(storage: &'a S) -> Box<dyn Iterator<Item = (K, V)> + 'a>;
+}
+
 // FIXME #1466 Remove this in favour of `decl_storage` macro.
 /// Declares strongly-typed wrappers around codec-compatible types in storage.
 #[macro_export]
@@ -523,7 +555,7 @@ macro_rules! __handle_wrap_internal {
 mod tests {
 	use std::collections::HashMap;
 	use std::cell::RefCell;
-	use codec::Codec;
+	use codec::{Decode, Encode};
 	use super::*;
 	use crate::rstd::marker::PhantomData;
 
@@ -532,11 +564,11 @@ mod tests {
 			self.borrow_mut().get(key).is_some()
 		}
 
-		fn get<T: Codec>(&self, key: &[u8]) -> Option<T> {
+		fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
 			self.borrow_mut().get(key).map(|v| T::decode(&mut &v[..]).unwrap())
 		}
 
-		fn put<T: Codec>(&self, key: &[u8], val: &T) {
+		fn put<T: Encode>(&self, key: &[u8], val: &T) {
 			self.borrow_mut().insert(key.to_owned(), val.encode());
 		}
 

substrate/srml/support/src/storage/mod.rs

@@ -19,7 +19,7 @@
 use crate::rstd::prelude::*;
 use crate::rstd::borrow::Borrow;
 use runtime_io::{self, twox_128};
-use crate::codec::{Codec, Decode, KeyedVec, Input};
+use crate::codec::{Codec, Encode, Decode, KeyedVec, Input};
 
 #[macro_use]
 pub mod generator;
@@ -39,7 +39,7 @@ impl<'a> Input for IncrementalInput<'a> {
 }
 
  /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
-pub fn get<T: Codec + Sized>(key: &[u8]) -> Option<T> {
+pub fn get<T: Decode + Sized>(key: &[u8]) -> Option<T> {
 	let key = twox_128(key);
 	runtime_io::read_storage(&key[..], &mut [0; 0][..], 0).map(|_| {
 		let mut input = IncrementalInput {
@@ -52,29 +52,29 @@ pub fn get<T: Codec + Sized>(key: &[u8]) -> Option<T> {
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
 /// explicit entry.
-pub fn get_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
+pub fn get_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
 	get(key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry.
-pub fn get_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
+pub fn get_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
 	get(key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry.
-pub fn get_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
 	get(key).unwrap_or_else(default_value)
 }
 
 /// Put `value` in storage under `key`.
-pub fn put<T: Codec>(key: &[u8], value: &T) {
+pub fn put<T: Encode>(key: &[u8], value: &T) {
 	value.using_encoded(|slice| runtime_io::set_storage(&twox_128(key)[..], slice));
 }
 
 /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
-pub fn take<T: Codec + Sized>(key: &[u8]) -> Option<T> {
+pub fn take<T: Decode + Sized>(key: &[u8]) -> Option<T> {
 	let r = get(key);
 	if r.is_some() {
 		kill(key);
@@ -84,19 +84,19 @@ pub fn take<T: Codec + Sized>(key: &[u8]) -> Option<T> {
 
 /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
 /// the default for its type.
-pub fn take_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
+pub fn take_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
 	take(key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
+pub fn take_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
 	take(key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+pub fn take_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
 	take(key).unwrap_or_else(default_value)
 }
 
@@ -129,12 +129,12 @@ impl crate::GenericStorage for RuntimeStorage {
 	}
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
-	fn get<T: Codec>(&self, key: &[u8]) -> Option<T> {
+	fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
 		super::storage::get(key)
 	}
 
 	/// Put a value in under a key.
-	fn put<T: Codec>(&self, key: &[u8], val: &T) {
+	fn put<T: Encode>(&self, key: &[u8], val: &T) {
 		super::storage::put(key, val)
 	}
 
@@ -144,7 +144,7 @@ impl crate::GenericStorage for RuntimeStorage {
 	}
 
 	/// Take a value from storage, deleting it after reading.
-	fn take<T: Codec>(&self, key: &[u8]) -> Option<T> {
+	fn take<T: Decode>(&self, key: &[u8]) -> Option<T> {
 		super::storage::take(key)
 	}
 }
@@ -336,6 +336,28 @@ impl<K: Codec, V: Codec, U> StorageMap<K, V> for U where U: generator::StorageMa
 	}
 }
 
+/// A storage map that can be enumerated.
+///
+/// Note that type is primarily useful for off-chain computations.
+/// Runtime implementors should avoid enumerating storage entries.
+pub trait EnumerableStorageMap<K: Codec, V: Codec>: StorageMap<K, V> {
+	/// Return current head element.
+	fn head() -> Option<K>;
+
+	/// Enumerate all elements in the map.
+	fn enumerate() -> Box<dyn Iterator<Item = (K, V)>>;
+}
+
+impl<K: Codec, V: Codec, U> EnumerableStorageMap<K, V> for U where U: generator::EnumerableStorageMap<K, V> {
+	fn head() -> Option<K> {
+		<U as generator::EnumerableStorageMap<K, V>>::head(&RuntimeStorage)
+	}
+
+	fn enumerate() -> Box<dyn Iterator<Item = (K, V)>> {
+		<U as generator::EnumerableStorageMap<K, V>>::enumerate(&RuntimeStorage)
+	}
+}
+
 /// A trait to conveniently store a vector of storable data.
 pub trait StorageVec {
 	type Item: Default + Sized + Codec;