storage doublemap in decl_storage (#1918)

* factorization

* introduce GenericUnhashedStorage

* implement generator and storage

* impl double map in storage macro

* improve StorageDoubleMapXX methods

* remove storage from example and impl test

* remove old comments

* wasm compatible

* improve imports

* rename storages

* update runtime impl version

* make code less verbose

* impl hash config for second key in double map

hash available are all of Hashable trait

* use double map in decl_storage for contract

* fix double map config issue

* add hasher into metadata

* update impl version and build wasm

* doc

* add attrs

* update metadata version

* update runtime version

* fix unused storage

substrate/srml/support/src/double_map.rs

@@ -31,7 +31,9 @@ use sr_std::borrow::Borrow;
 /// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts.
 /// The first part is a hash of a concatenation of the `PREFIX` and `Key1`. And the second part
 /// is a hash of a `Key2`.
-pub trait StorageDoubleMap {
+///
+/// Hasher are implemented in derive_key* methods.
+pub trait StorageDoubleMapWithHasher {
 	type Key1: Codec;
 	type Key2: Codec;
 	type Value: Codec + Default;

substrate/srml/support/src/hashable.rs

@@ -27,12 +27,12 @@ pub trait Hashable: Sized {
 
 impl<T: Codec> Hashable for T {
 	fn blake2_256(&self) -> [u8; 32] {
-		blake2_256(&self.encode())
+		self.using_encoded(blake2_256)
 	}
 	fn twox_128(&self) -> [u8; 16] {
-		twox_128(&self.encode())
+		self.using_encoded(twox_128)
 	}
 	fn twox_256(&self) -> [u8; 32] {
-		twox_256(&self.encode())
+		self.using_encoded(twox_256)
 	}
 }

substrate/srml/support/src/lib.rs

@@ -36,6 +36,7 @@ pub use paste;
 pub use sr_primitives as runtime_primitives;
 
 pub use self::storage::generator::Storage as GenericStorage;
+pub use self::storage::unhashed::generator::UnhashedStorage as GenericUnhashedStorage;
 
 #[macro_use]
 pub mod dispatch;
@@ -55,10 +56,10 @@ pub mod inherent;
 mod double_map;
 pub mod traits;
 
-pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap, EnumerableStorageMap};
+pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap, EnumerableStorageMap, StorageDoubleMap};
 pub use self::hashable::Hashable;
 pub use self::dispatch::{Parameter, Dispatchable, Callable, IsSubType};
-pub use self::double_map::StorageDoubleMap;
+pub use self::double_map::StorageDoubleMapWithHasher;
 pub use runtime_io::print;
 
 #[doc(inline)]
@@ -140,6 +141,12 @@ mod tests {
 	use parity_codec::Codec;
 	use runtime_io::{with_externalities, Blake2Hasher};
 	use runtime_primitives::BuildStorage;
+	pub use srml_metadata::{
+		DecodeDifferent, StorageMetadata, StorageFunctionMetadata,
+		StorageFunctionType, StorageFunctionModifier,
+		DefaultByte, DefaultByteGetter,
+	};
+	pub use rstd::marker::PhantomData;
 
 	pub trait Trait {
 		type BlockNumber: Codec + Default;
@@ -164,6 +171,10 @@ mod tests {
 			pub Data get(data) build(|_| vec![(15u32, 42u64)]): linked_map u32 => u64;
 			pub GenericData get(generic_data): linked_map T::BlockNumber => T::BlockNumber;
 			pub GenericData2 get(generic_data2): linked_map T::BlockNumber => Option<T::BlockNumber>;
+
+			pub DataDM config(test_config) build(|_| vec![(15u32, 16u32, 42u64)]): double_map u32, blake2_256(u32) => u64;
+			pub GenericDataDM: double_map T::BlockNumber, twox_128(T::BlockNumber) => T::BlockNumber;
+			pub GenericData2DM: double_map T::BlockNumber, twox_256(T::BlockNumber) => Option<T::BlockNumber>;
 		}
 	}
 
@@ -180,7 +191,7 @@ mod tests {
 	type Map = Data<Test>;
 
 	#[test]
-	fn basic_insert_remove_should_work() {
+	fn linked_map_basic_insert_remove_should_work() {
 		with_externalities(&mut new_test_ext(), || {
 			// initialised during genesis
 			assert_eq!(Map::get(&15u32), 42u64);
@@ -206,7 +217,7 @@ mod tests {
 	}
 
 	#[test]
-	fn enumeration_and_head_should_work() {
+	fn linked_map_enumeration_and_head_should_work() {
 		with_externalities(&mut new_test_ext(), || {
 			assert_eq!(Map::head(), Some(15));
 			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(15, 42)]);
@@ -257,4 +268,128 @@ mod tests {
 		});
 	}
 
+	#[test]
+	fn double_map_basic_insert_remove_remove_prefix_should_work() {
+		with_externalities(&mut new_test_ext(), || {
+			type DoubleMap = DataDM<Test>;
+			// initialised during genesis
+			assert_eq!(DoubleMap::get(&15u32, &16u32), 42u64);
+
+			// get / insert / take
+			let key1 = 17u32;
+			let key2 = 18u32;
+			assert_eq!(DoubleMap::get(key1, key2), 0u64);
+			DoubleMap::insert(key1, key2, 4u64);
+			assert_eq!(DoubleMap::get(key1, key2), 4u64);
+			assert_eq!(DoubleMap::take(key1, key2), 4u64);
+			assert_eq!(DoubleMap::get(key1, key2), 0u64);
+
+			// mutate
+			DoubleMap::mutate(key1, key2, |val| {
+				*val = 15;
+			});
+			assert_eq!(DoubleMap::get(key1, key2), 15u64);
+
+			// remove
+			DoubleMap::remove(key1, key2);
+			assert_eq!(DoubleMap::get(key1, key2), 0u64);
+
+			// remove prefix
+			DoubleMap::insert(key1, key2, 4u64);
+			DoubleMap::insert(key1, key2+1, 4u64);
+			DoubleMap::insert(key1+1, key2, 4u64);
+			DoubleMap::insert(key1+1, key2+1, 4u64);
+			DoubleMap::remove_prefix(key1);
+			assert_eq!(DoubleMap::get(key1, key2), 0u64);
+			assert_eq!(DoubleMap::get(key1, key2+1), 0u64);
+			assert_eq!(DoubleMap::get(key1+1, key2), 4u64);
+			assert_eq!(DoubleMap::get(key1+1, key2+1), 4u64);
+		});
+	}
+
+	const EXPECTED_METADATA: StorageMetadata = StorageMetadata {
+		functions: DecodeDifferent::Encode(&[
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("Data"),
+				modifier: StorageFunctionModifier::Default,
+				ty: StorageFunctionType::Map{
+					key: DecodeDifferent::Encode("u32"), value: DecodeDifferent::Encode("u64"), is_linked: true
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructData(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("GenericData"),
+				modifier: StorageFunctionModifier::Default,
+				ty: StorageFunctionType::Map{
+					key: DecodeDifferent::Encode("T::BlockNumber"), value: DecodeDifferent::Encode("T::BlockNumber"), is_linked: true
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructGenericData(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("GenericData2"),
+				modifier: StorageFunctionModifier::Optional,
+				ty: StorageFunctionType::Map{
+					key: DecodeDifferent::Encode("T::BlockNumber"), value: DecodeDifferent::Encode("T::BlockNumber"), is_linked: true
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructGenericData2(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("DataDM"),
+				modifier: StorageFunctionModifier::Default,
+				ty: StorageFunctionType::DoubleMap{
+					key1: DecodeDifferent::Encode("u32"),
+					key2: DecodeDifferent::Encode("u32"),
+					value: DecodeDifferent::Encode("u64"),
+					key2_hasher: DecodeDifferent::Encode("blake2_256"),
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructDataDM(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("GenericDataDM"),
+				modifier: StorageFunctionModifier::Default,
+				ty: StorageFunctionType::DoubleMap{
+					key1: DecodeDifferent::Encode("T::BlockNumber"),
+					key2: DecodeDifferent::Encode("T::BlockNumber"),
+					value: DecodeDifferent::Encode("T::BlockNumber"),
+					key2_hasher: DecodeDifferent::Encode("twox_128"),
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructGenericDataDM(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+			StorageFunctionMetadata {
+				name: DecodeDifferent::Encode("GenericData2DM"),
+				modifier: StorageFunctionModifier::Optional,
+				ty: StorageFunctionType::DoubleMap{
+					key1: DecodeDifferent::Encode("T::BlockNumber"),
+					key2: DecodeDifferent::Encode("T::BlockNumber"),
+					value: DecodeDifferent::Encode("T::BlockNumber"),
+					key2_hasher: DecodeDifferent::Encode("twox_256"),
+				},
+				default: DecodeDifferent::Encode(
+					DefaultByteGetter(&__GetByteStructGenericData2DM(PhantomData::<Test>))
+				),
+				documentation: DecodeDifferent::Encode(&[]),
+			},
+		])
+	};
+
+	#[test]
+	fn store_metadata() {
+		let metadata = Module::<Test>::store_metadata();
+		assert_eq!(EXPECTED_METADATA, metadata);
+	}
 }

substrate/srml/support/src/metadata.rs

@@ -16,7 +16,7 @@
 
 pub use srml_metadata::{
 	DecodeDifferent, FnEncode, RuntimeMetadata,
-	ModuleMetadata, RuntimeMetadataV2,
+	ModuleMetadata, RuntimeMetadataV3,
 	DefaultByteGetter, RuntimeMetadataPrefixed,
 };
 
@@ -36,8 +36,8 @@ macro_rules! impl_runtime_metadata {
 	) => {
 		impl $runtime {
 			pub fn metadata() -> $crate::metadata::RuntimeMetadataPrefixed {
-				$crate::metadata::RuntimeMetadata::V2 (
-					$crate::metadata::RuntimeMetadataV2 {
+				$crate::metadata::RuntimeMetadata::V3 (
+					$crate::metadata::RuntimeMetadataV3 {
 						modules: $crate::__runtime_modules_to_metadata!($runtime;; $( $rest )*),
 					}
 				).into()
@@ -377,8 +377,8 @@ mod tests {
 			event_module2::Module with Event Storage Call,
 	);
 
-	const EXPECTED_METADATA: RuntimeMetadata = RuntimeMetadata::V2(
-		RuntimeMetadataV2 {
+	const EXPECTED_METADATA: RuntimeMetadata = RuntimeMetadata::V3(
+		RuntimeMetadataV3 {
 		modules: DecodeDifferent::Encode(&[
 			ModuleMetadata {
 				name: DecodeDifferent::Encode("system"),

substrate/srml/support/src/storage/generator.rs

@@ -48,6 +48,8 @@
 
 use crate::codec;
 use crate::rstd::vec::Vec;
+#[cfg(feature = "std")]
+use crate::storage::unhashed::generator::UnhashedStorage;
 #[doc(hidden)]
 pub use crate::rstd::borrow::Borrow;
 #[doc(hidden)]
@@ -101,20 +103,19 @@ pub trait Storage {
 #[cfg(feature = "std")]
 impl<S: sr_primitives::BuildStorage> Storage for (crate::rstd::cell::RefCell<&mut sr_primitives::StorageOverlay>, PhantomData<S>) {
 	fn exists(&self, key: &[u8]) -> bool {
-		self.0.borrow().contains_key(S::hash(key).as_ref())
+		UnhashedStorage::exists(self, &S::hash(key))
 	}
 
 	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
-		self.0.borrow().get(S::hash(key).as_ref())
-			.map(|x| codec::Decode::decode(&mut x.as_slice()).expect("Unable to decode expected type."))
+		UnhashedStorage::get(self, &S::hash(key))
 	}
 
 	fn put<T: codec::Encode>(&self, key: &[u8], val: &T) {
-		self.0.borrow_mut().insert(S::hash(key).to_vec(), codec::Encode::encode(val));
+		UnhashedStorage::put(self, &S::hash(key), val)
 	}
 
 	fn kill(&self, key: &[u8]) {
-		self.0.borrow_mut().remove(S::hash(key).as_ref());
+		UnhashedStorage::kill(self, &S::hash(key))
 	}
 }
 

substrate/srml/support/src/storage/mod.rs

@@ -23,6 +23,7 @@ use crate::codec::{Codec, Encode, Decode, KeyedVec, Input};
 
 #[macro_use]
 pub mod generator;
+pub mod unhashed;
 
 struct IncrementalInput<'a> {
 	key: &'a [u8],
@@ -56,84 +57,73 @@ impl<'a> Input for IncrementalChildInput<'a> {
 
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
 pub fn get<T: Decode + Sized>(key: &[u8]) -> Option<T> {
-	let key = twox_128(key);
-	runtime_io::read_storage(&key[..], &mut [0; 0][..], 0).map(|_| {
-		let mut input = IncrementalInput {
-			key: &key[..],
-			pos: 0,
-		};
-		Decode::decode(&mut input).expect("storage is not null, therefore must be a valid type")
-	})
+	unhashed::get(&twox_128(key))
 }
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
 /// explicit entry.
 pub fn get_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
-	get(key).unwrap_or_else(Default::default)
+	unhashed::get_or_default(&twox_128(key))
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry.
 pub fn get_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
-	get(key).unwrap_or(default_value)
+	unhashed::get_or(&twox_128(key), default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry.
 pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
-	get(key).unwrap_or_else(default_value)
+	unhashed::get_or_else(&twox_128(key), default_value)
 }
 
 /// Put `value` in storage under `key`.
 pub fn put<T: Encode>(key: &[u8], value: &T) {
-	value.using_encoded(|slice| runtime_io::set_storage(&twox_128(key)[..], slice));
+	unhashed::put(&twox_128(key), value)
 }
 
 /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
 pub fn take<T: Decode + Sized>(key: &[u8]) -> Option<T> {
-	let r = get(key);
-	if r.is_some() {
-		kill(key);
-	}
-	r
+	unhashed::take(&twox_128(key))
 }
 
 /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
 /// the default for its type.
 pub fn take_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
-	take(key).unwrap_or_else(Default::default)
+	unhashed::take_or_default(&twox_128(key))
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
 pub fn take_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
-	take(key).unwrap_or(default_value)
+	unhashed::take_or(&twox_128(key), default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
 pub fn take_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
-	take(key).unwrap_or_else(default_value)
+	unhashed::take_or_else(&twox_128(key), default_value)
 }
 
 /// Check to see if `key` has an explicit entry in storage.
 pub fn exists(key: &[u8]) -> bool {
-	runtime_io::exists_storage(&twox_128(key)[..])
+	unhashed::exists(&twox_128(key))
 }
 
 /// Ensure `key` has no explicit entry in storage.
 pub fn kill(key: &[u8]) {
-	runtime_io::clear_storage(&twox_128(key)[..]);
+	unhashed::kill(&twox_128(key))
 }
 
 /// Get a Vec of bytes from storage.
 pub fn get_raw(key: &[u8]) -> Option<Vec<u8>> {
-	runtime_io::storage(&twox_128(key)[..])
+	unhashed::get_raw(&twox_128(key))
 }
 
 /// Put a raw byte slice into storage.
 pub fn put_raw(key: &[u8], value: &[u8]) {
-	runtime_io::set_storage(&twox_128(key)[..], value)
+	unhashed::put_raw(&twox_128(key), value)
 }
 
 /// The underlying runtime storage.
@@ -141,27 +131,58 @@ pub struct RuntimeStorage;
 
 impl crate::GenericStorage for RuntimeStorage {
 	fn exists(&self, key: &[u8]) -> bool {
-		super::storage::exists(key)
+		exists(key)
 	}
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
 	fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
-		super::storage::get(key)
+		get(key)
 	}
 
 	/// Put a value in under a key.
 	fn put<T: Encode>(&self, key: &[u8], val: &T) {
-		super::storage::put(key, val)
+		put(key, val)
 	}
 
 	/// Remove the bytes of a key from storage.
 	fn kill(&self, key: &[u8]) {
-		super::storage::kill(key)
+		kill(key)
 	}
 
 	/// Take a value from storage, deleting it after reading.
 	fn take<T: Decode>(&self, key: &[u8]) -> Option<T> {
-		super::storage::take(key)
+		take(key)
+	}
+}
+
+impl crate::GenericUnhashedStorage for RuntimeStorage {
+	fn exists(&self, key: &[u8]) -> bool {
+		unhashed::exists(key)
+	}
+
+	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
+	fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
+		unhashed::get(key)
+	}
+
+	/// Put a value in under a key.
+	fn put<T: Encode>(&self, key: &[u8], val: &T) {
+		unhashed::put(key, val)
+	}
+
+	/// Remove the bytes of a key from storage.
+	fn kill(&self, key: &[u8]) {
+		unhashed::kill(key)
+	}
+
+	/// Remove the bytes of a key from storage.
+	fn kill_prefix(&self, prefix: &[u8]) {
+		unhashed::kill_prefix(prefix)
+	}
+
+	/// Take a value from storage, deleting it after reading.
+	fn take<T: Decode>(&self, key: &[u8]) -> Option<T> {
+		unhashed::take(key)
 	}
 }
 
@@ -374,6 +395,109 @@ impl<K: Codec, V: Codec, U> EnumerableStorageMap<K, V> for U where U: generator:
 	}
 }
 
+/// An implementation of a map with a two keys.
+///
+/// It provides an important ability to efficiently remove all entries
+/// that have a common first key.
+///
+/// # Mapping of keys to a storage path
+///
+/// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts.
+/// The first part is a hash of a concatenation of the `PREFIX` and `Key1`. And the second part
+/// is a hash of a `Key2`.
+///
+/// /!\ be careful while choosing the Hash, indeed malicious could craft second keys to lower the trie.
+pub trait StorageDoubleMap<K1: Codec, K2: Codec, V: Codec> {
+	/// The type that get/take returns.
+	type Query;
+
+	/// Get the prefix key in storage.
+	fn prefix() -> &'static [u8];
+
+	/// Get the storage key used to fetch a value corresponding to a specific key.
+	fn key_for<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Vec<u8>;
+
+	/// Get the storage prefix used to fetch keys corresponding to a specific key1.
+	fn prefix_for<KArg1: Borrow<K1>>(k1: KArg1) -> Vec<u8>;
+
+	/// true if the value is defined in storage.
+	fn exists<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> bool;
+
+	/// Load the value associated with the given key from the map.
+	fn get<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Self::Query;
+
+	/// Take the value under a key.
+	fn take<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Self::Query;
+
+	/// Store a value to be associated with the given key from the map.
+	fn insert<KArg1: Borrow<K1>, KArg2: Borrow<K2>, VArg: Borrow<V>>(k1: KArg1, k2: KArg2, val: VArg);
+
+	/// Remove the value under a key.
+	fn remove<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2);
+
+	/// Removes all entries that shares the `k1` as the first key.
+	fn remove_prefix<KArg1: Borrow<K1>>(k1: KArg1);
+
+	/// Mutate the value under a key.
+	fn mutate<KArg1, KArg2, R, F>(k1: KArg1, k2: KArg2, f: F) -> R
+	where
+		KArg1: Borrow<K1>,
+		KArg2: Borrow<K2>,
+		F: FnOnce(&mut Self::Query) -> R;
+}
+
+impl<K1: Codec, K2: Codec, V: Codec, U> StorageDoubleMap<K1, K2, V> for U
+where
+	U: unhashed::generator::StorageDoubleMap<K1, K2, V>
+{
+	type Query = U::Query;
+
+	fn prefix() -> &'static [u8] {
+		<U as unhashed::generator::StorageDoubleMap<K1, K2, V>>::prefix()
+	}
+
+	fn key_for<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Vec<u8> {
+		<U as unhashed::generator::StorageDoubleMap<K1, K2, V>>::key_for(k1.borrow(), k2.borrow())
+	}
+
+	fn prefix_for<KArg1: Borrow<K1>>(k1: KArg1) -> Vec<u8> {
+		<U as unhashed::generator::StorageDoubleMap<K1, K2, V>>::prefix_for(k1.borrow())
+	}
+
+	fn exists<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> bool {
+		U::exists(k1.borrow(), k2.borrow(), &RuntimeStorage)
+	}
+
+	fn get<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Self::Query {
+		U::get(k1.borrow(), k2.borrow(), &RuntimeStorage)
+	}
+
+	fn take<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) -> Self::Query {
+		U::take(k1.borrow(), k2.borrow(), &RuntimeStorage)
+	}
+
+	fn insert<KArg1: Borrow<K1>, KArg2: Borrow<K2>, VArg: Borrow<V>>(k1: KArg1, k2: KArg2, val: VArg) {
+		U::insert(k1.borrow(), k2.borrow(), val.borrow(), &RuntimeStorage)
+	}
+
+	fn remove<KArg1: Borrow<K1>, KArg2: Borrow<K2>>(k1: KArg1, k2: KArg2) {
+		U::remove(k1.borrow(), k2.borrow(), &RuntimeStorage)
+	}
+
+	fn remove_prefix<KArg1: Borrow<K1>>(k1: KArg1) {
+		U::remove_prefix(k1.borrow(), &RuntimeStorage)
+	}
+
+	fn mutate<KArg1, KArg2, R, F>(k1: KArg1, k2: KArg2, f: F) -> R
+	where
+		KArg1: Borrow<K1>,
+		KArg2: Borrow<K2>,
+		F: FnOnce(&mut Self::Query) -> R
+	{
+		U::mutate(k1.borrow(), k2.borrow(), f, &RuntimeStorage)
+	}
+}
+
 /// A trait to conveniently store a vector of storable data.
 pub trait StorageVec {
 	type Item: Default + Sized + Codec;
@@ -433,149 +557,6 @@ pub trait StorageVec {
 	}
 }
 
-pub mod unhashed {
-	use crate::rstd::borrow::Borrow;
-	use super::{runtime_io, Codec, Decode, KeyedVec, Vec, IncrementalInput};
-
-	/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
-	pub fn get<T: Codec + Sized>(key: &[u8]) -> Option<T> {
-		runtime_io::read_storage(key, &mut [0; 0][..], 0).map(|_| {
-			let mut input = IncrementalInput {
-				key,
-				pos: 0,
-			};
-			Decode::decode(&mut input).expect("storage is not null, therefore must be a valid type")
-		})
-	}
-
-	/// Return the value of the item in storage under `key`, or the type's default if there is no
-	/// explicit entry.
-	pub fn get_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
-		get(key).unwrap_or_else(Default::default)
-	}
-
-	/// Return the value of the item in storage under `key`, or `default_value` if there is no
-	/// explicit entry.
-	pub fn get_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
-		get(key).unwrap_or(default_value)
-	}
-
-	/// Return the value of the item in storage under `key`, or `default_value()` if there is no
-	/// explicit entry.
-	pub fn get_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
-		get(key).unwrap_or_else(default_value)
-	}
-
-	/// Put `value` in storage under `key`.
-	pub fn put<T: Codec>(key: &[u8], value: &T) {
-		value.using_encoded(|slice| runtime_io::set_storage(key, slice));
-	}
-
-	/// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
-	pub fn take<T: Codec + Sized>(key: &[u8]) -> Option<T> {
-		let r = get(key);
-		if r.is_some() {
-			kill(key);
-		}
-		r
-	}
-
-	/// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
-	/// the default for its type.
-	pub fn take_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
-		take(key).unwrap_or_else(Default::default)
-	}
-
-	/// Return the value of the item in storage under `key`, or `default_value` if there is no
-	/// explicit entry. Ensure there is no explicit entry on return.
-	pub fn take_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
-		take(key).unwrap_or(default_value)
-	}
-
-	/// Return the value of the item in storage under `key`, or `default_value()` if there is no
-	/// explicit entry. Ensure there is no explicit entry on return.
-	pub fn take_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
-		take(key).unwrap_or_else(default_value)
-	}
-
-	/// Check to see if `key` has an explicit entry in storage.
-	pub fn exists(key: &[u8]) -> bool {
-		runtime_io::read_storage(key, &mut [0;0][..], 0).is_some()
-	}
-
-	/// Ensure `key` has no explicit entry in storage.
-	pub fn kill(key: &[u8]) {
-		runtime_io::clear_storage(key);
-	}
-
-	/// Ensure keys with the given `prefix` have no entries in storage.
-	pub fn kill_prefix(prefix: &[u8]) {
-		runtime_io::clear_prefix(prefix);
-	}
-
-	/// Get a Vec of bytes from storage.
-	pub fn get_raw(key: &[u8]) -> Option<Vec<u8>> {
-		runtime_io::storage(key)
-	}
-
-	/// Put a raw byte slice into storage.
-	pub fn put_raw(key: &[u8], value: &[u8]) {
-		runtime_io::set_storage(key, value)
-	}
-
-	/// A trait to conveniently store a vector of storable data.
-	pub trait StorageVec {
-		type Item: Default + Sized + Codec;
-		const PREFIX: &'static [u8];
-
-		/// Get the current set of items.
-		fn items() -> Vec<Self::Item> {
-			(0..Self::count()).into_iter().map(Self::item).collect()
-		}
-
-		/// Set the current set of items.
-		fn set_items<I, T>(items: I)
-			where
-				I: IntoIterator<Item=T>,
-				T: Borrow<Self::Item>,
-		{
-			let mut count: u32 = 0;
-
-			for i in items.into_iter() {
-				put(&count.to_keyed_vec(Self::PREFIX), i.borrow());
-				count = count.checked_add(1).expect("exceeded runtime storage capacity");
-			}
-
-			Self::set_count(count);
-		}
-
-		fn set_item(index: u32, item: &Self::Item) {
-			if index < Self::count() {
-				put(&index.to_keyed_vec(Self::PREFIX), item);
-			}
-		}
-
-		fn clear_item(index: u32) {
-			if index < Self::count() {
-				kill(&index.to_keyed_vec(Self::PREFIX));
-			}
-		}
-
-		fn item(index: u32) -> Self::Item {
-			get_or_default(&index.to_keyed_vec(Self::PREFIX))
-		}
-
-		fn set_count(count: u32) {
-			(count..Self::count()).for_each(Self::clear_item);
-			put(&b"len".to_keyed_vec(Self::PREFIX), &count);
-		}
-
-		fn count() -> u32 {
-			get_or_default(&b"len".to_keyed_vec(Self::PREFIX))
-		}
-	}
-}
-
 /// child storage NOTE could replace unhashed by having only one kind of storage (root being null storage
 /// key (storage_key can become Option<&[u8]>).
 /// This module is a currently only a variant of unhashed with additional `storage_key`.

substrate/srml/support/src/storage/unhashed/generator.rs

@@ -0,0 +1,144 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::codec;
+use runtime_io::twox_128;
+use crate::rstd::vec::Vec;
+
+/// Abstraction around storage with unhashed access.
+pub trait UnhashedStorage {
+	/// true if the key exists in storage.
+	fn exists(&self, key: &[u8]) -> bool;
+
+	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T>;
+
+	/// Load the bytes of a key from storage. Can panic if the type is incorrect. Will panic if
+	/// it's not there.
+	fn require<T: codec::Decode>(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") }
+
+	/// Load the bytes of a key from storage. Can panic if the type is incorrect. The type's
+	/// default is returned if it's not there.
+	fn get_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() }
+
+	/// Put a value in under a key.
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T);
+
+	/// Remove the bytes of a key from storage.
+	fn kill(&self, key: &[u8]);
+
+	/// Remove the bytes of a key from storage.
+	fn kill_prefix(&self, prefix: &[u8]);
+
+	/// Take a value from storage, deleting it after reading.
+	fn take<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
+		let value = self.get(key);
+		self.kill(key);
+		value
+	}
+
+	/// Take a value from storage, deleting it after reading.
+	fn take_or_panic<T: codec::Decode>(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") }
+
+	/// Take a value from storage, deleting it after reading.
+	fn take_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() }
+}
+
+// We use a construct like this during when genesis storage is being built.
+#[cfg(feature = "std")]
+impl<H> UnhashedStorage for (crate::rstd::cell::RefCell<&mut sr_primitives::StorageOverlay>, H) {
+	fn exists(&self, key: &[u8]) -> bool {
+		self.0.borrow().contains_key(key)
+	}
+
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
+		self.0.borrow().get(key)
+			.map(|x| codec::Decode::decode(&mut x.as_slice()).expect("Unable to decode expected type."))
+	}
+
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T) {
+		self.0.borrow_mut().insert(key.to_vec(), codec::Encode::encode(val));
+	}
+
+	fn kill(&self, key: &[u8]) {
+		self.0.borrow_mut().remove(key);
+	}
+
+	fn kill_prefix(&self, prefix: &[u8]) {
+		self.0.borrow_mut().retain(|key, _| {
+			!key.starts_with(prefix)
+		})
+	}
+}
+
+/// An implementation of a map with a two keys.
+///
+/// It provides an important ability to efficiently remove all entries
+/// that have a common first key.
+///
+/// # Mapping of keys to a storage path
+///
+/// The storage key (i.e. the key under which the `Value` will be stored) is created from two parts.
+/// The first part is a hash of a concatenation of the `PREFIX` and `Key1`. And the second part
+/// is a hash of a `Key2`.
+///
+/// /!\ be careful while choosing the Hash, indeed malicious could craft second keys to lower the trie.
+pub trait StorageDoubleMap<K1: codec::Codec, K2: codec::Codec, V: codec::Codec> {
+	/// The type that get/take returns.
+	type Query;
+
+	/// Get the prefix key in storage.
+	fn prefix() -> &'static [u8];
+
+	/// Get the storage key used to fetch a value corresponding to a specific key.
+	fn key_for(k1: &K1, k2: &K2) -> Vec<u8>;
+
+	/// Get the storage prefix used to fetch keys corresponding to a specific key1.
+	fn prefix_for(k1: &K1) -> Vec<u8> {
+		let mut key = Self::prefix().to_vec();
+		codec::Encode::encode_to(k1, &mut key);
+		twox_128(&key).to_vec()
+	}
+
+	/// true if the value is defined in storage.
+	fn exists<S: UnhashedStorage>(k1: &K1, k2: &K2, storage: &S) -> bool {
+		storage.exists(&Self::key_for(k1, k2))
+	}
+
+	/// Load the value associated with the given key from the map.
+	fn get<S: UnhashedStorage>(k1: &K1, k2: &K2, storage: &S) -> Self::Query;
+
+	/// Take the value under a key.
+	fn take<S: UnhashedStorage>(k1: &K1, k2: &K2, storage: &S) -> Self::Query;
+
+	/// Store a value to be associated with the given key from the map.
+	fn insert<S: UnhashedStorage>(k1: &K1, k2: &K2, val: &V, storage: &S) {
+		storage.put(&Self::key_for(k1, k2), val);
+	}
+
+	/// Remove the value under a key.
+	fn remove<S: UnhashedStorage>(k1: &K1, k2: &K2, storage: &S) {
+		storage.kill(&Self::key_for(k1, k2));
+	}
+
+	/// Removes all entries that shares the `k1` as the first key.
+	fn remove_prefix<S: UnhashedStorage>(k1: &K1, storage: &S) {
+		storage.kill_prefix(&Self::prefix_for(k1));
+	}
+
+	/// Mutate the value under a key.
+	fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: UnhashedStorage>(k1: &K1, k2: &K2, f: F, storage: &S) -> R;
+}

substrate/srml/support/src/storage/unhashed/mod.rs

@@ -0,0 +1,160 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Operation on unhashed runtime storage
+
+use crate::rstd::borrow::Borrow;
+use super::{runtime_io, Codec, Encode, Decode, KeyedVec, Vec, IncrementalInput};
+
+pub mod generator;
+
+/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
+pub fn get<T: Decode + Sized>(key: &[u8]) -> Option<T> {
+	runtime_io::read_storage(key, &mut [0; 0][..], 0).map(|_| {
+		let mut input = IncrementalInput {
+			key,
+			pos: 0,
+		};
+		Decode::decode(&mut input).expect("storage is not null, therefore must be a valid type")
+	})
+}
+
+/// Return the value of the item in storage under `key`, or the type's default if there is no
+/// explicit entry.
+pub fn get_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
+	get(key).unwrap_or_else(Default::default)
+}
+
+/// Return the value of the item in storage under `key`, or `default_value` if there is no
+/// explicit entry.
+pub fn get_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
+	get(key).unwrap_or(default_value)
+}
+
+/// Return the value of the item in storage under `key`, or `default_value()` if there is no
+/// explicit entry.
+pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+	get(key).unwrap_or_else(default_value)
+}
+
+/// Put `value` in storage under `key`.
+pub fn put<T: Encode>(key: &[u8], value: &T) {
+	value.using_encoded(|slice| runtime_io::set_storage(key, slice));
+}
+
+/// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
+pub fn take<T: Decode + Sized>(key: &[u8]) -> Option<T> {
+	let r = get(key);
+	if r.is_some() {
+		kill(key);
+	}
+	r
+}
+
+/// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
+/// the default for its type.
+pub fn take_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
+	take(key).unwrap_or_else(Default::default)
+}
+
+/// Return the value of the item in storage under `key`, or `default_value` if there is no
+/// explicit entry. Ensure there is no explicit entry on return.
+pub fn take_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
+	take(key).unwrap_or(default_value)
+}
+
+/// Return the value of the item in storage under `key`, or `default_value()` if there is no
+/// explicit entry. Ensure there is no explicit entry on return.
+pub fn take_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+	take(key).unwrap_or_else(default_value)
+}
+
+/// Check to see if `key` has an explicit entry in storage.
+pub fn exists(key: &[u8]) -> bool {
+	runtime_io::read_storage(key, &mut [0;0][..], 0).is_some()
+}
+
+/// Ensure `key` has no explicit entry in storage.
+pub fn kill(key: &[u8]) {
+	runtime_io::clear_storage(key);
+}
+
+/// Ensure keys with the given `prefix` have no entries in storage.
+pub fn kill_prefix(prefix: &[u8]) {
+	runtime_io::clear_prefix(prefix);
+}
+
+/// Get a Vec of bytes from storage.
+pub fn get_raw(key: &[u8]) -> Option<Vec<u8>> {
+	runtime_io::storage(key)
+}
+
+/// Put a raw byte slice into storage.
+pub fn put_raw(key: &[u8], value: &[u8]) {
+	runtime_io::set_storage(key, value)
+}
+
+/// A trait to conveniently store a vector of storable data.
+pub trait StorageVec {
+	type Item: Default + Sized + Codec;
+	const PREFIX: &'static [u8];
+
+	/// Get the current set of items.
+	fn items() -> Vec<Self::Item> {
+		(0..Self::count()).into_iter().map(Self::item).collect()
+	}
+
+	/// Set the current set of items.
+	fn set_items<I, T>(items: I)
+		where
+			I: IntoIterator<Item=T>,
+			T: Borrow<Self::Item>,
+	{
+		let mut count: u32 = 0;
+
+		for i in items.into_iter() {
+			put(&count.to_keyed_vec(Self::PREFIX), i.borrow());
+			count = count.checked_add(1).expect("exceeded runtime storage capacity");
+		}
+
+		Self::set_count(count);
+	}
+
+	fn set_item(index: u32, item: &Self::Item) {
+		if index < Self::count() {
+			put(&index.to_keyed_vec(Self::PREFIX), item);
+		}
+	}
+
+	fn clear_item(index: u32) {
+		if index < Self::count() {
+			kill(&index.to_keyed_vec(Self::PREFIX));
+		}
+	}
+
+	fn item(index: u32) -> Self::Item {
+		get_or_default(&index.to_keyed_vec(Self::PREFIX))
+	}
+
+	fn set_count(count: u32) {
+		(count..Self::count()).for_each(Self::clear_item);
+		put(&b"len".to_keyed_vec(Self::PREFIX), &count);
+	}
+
+	fn count() -> u32 {
+		get_or_default(&b"len".to_keyed_vec(Self::PREFIX))
+	}
+}