EnumerableStorageMap (#1763)

* Refactor decl_storage a bit to allow easier impl of linked map.

* A bunch of refactorings for storage generation.

- Rename StorageMap and ChildrenStorageMap to avoid confusion with generator::StorageMap.
- Separate implementation from the procedural macro code to clean it up.
- Make sure that genesis is initialised using the `StorageValue/StorageMap`
  generated implementations instead of going RAW.

* WiP: Writing test.

* Basic implementation.

* Implement enumeration.

* Fix non-std issues.

* fix warning

* Fix test-client.

* Address review grumbles - part 1

* Avoid cloning the key, relax Storage requirements.

* Rebuild runtime.

* Remove dangling todo.

substrate/core/client/db/src/lib.rs

@@ -44,7 +44,7 @@ use trie::MemoryDB;
 use parking_lot::RwLock;
 use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash};
 use primitives::storage::well_known_keys;
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero, Digest, DigestItem, AuthorityIdFor};
 use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
@@ -307,7 +307,7 @@ where Block: BlockT<Hash=H256>,
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, mut top: StorageMap, children: ChildrenStorageMap) -> Result<H256, client::error::Error> {
+	fn reset_storage(&mut self, mut top: StorageOverlay, children: ChildrenStorageOverlay) -> Result<H256, client::error::Error> {
 
 		if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 			return Err(client::error::ErrorKind::GenesisInvalid.into());

substrate/core/client/src/backend.rs

@@ -18,7 +18,7 @@
 
 use crate::error;
 use primitives::ChangesTrieConfiguration;
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{AuthorityIdFor, Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
 use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
@@ -79,7 +79,7 @@ pub trait BlockImportOperation<Block, H> where
 	/// Inject storage data into the database.
 	fn update_db_storage(&mut self, update: <Self::State as StateBackend<H>>::Transaction) -> error::Result<()>;
 	/// Inject storage data into the database replacing any existing data.
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out>;
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out>;
 	/// Set top level storage changes.
 	fn update_storage(&mut self, update: Vec<(Vec<u8>, Option<Vec<u8>>)>) -> error::Result<()>;
 	/// Inject changes trie data into the database.

substrate/core/client/src/in_mem.rs

@@ -26,7 +26,7 @@ use primitives::{ChangesTrieConfiguration, storage::well_known_keys};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
 	NumberFor, As, Digest, DigestItem, AuthorityIdFor};
-use runtime_primitives::{Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay};
 use crate::blockchain::{self, BlockStatus, HeaderBackend};
 use state_machine::backend::{Backend as StateBackend, InMemory, Consolidate};
 use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId};
@@ -490,7 +490,7 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, mut top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out> {
+	fn reset_storage(&mut self, mut top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		let mut transaction: Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> = Default::default();
@@ -753,7 +753,7 @@ pub fn cache_authorities_at<Block: BlockT>(
 }
 
 /// Check that genesis storage is valid.
-pub fn check_genesis_storage(top: &StorageMap, children: &ChildrenStorageMap) -> error::Result<()> {
+pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOverlay) -> error::Result<()> {
 	if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 		return Err(error::ErrorKind::GenesisInvalid.into());
 	}

substrate/core/client/src/light/backend.rs

@@ -22,7 +22,7 @@ use std::sync::{Arc, Weak};
 use futures::{Future, IntoFuture};
 use parking_lot::RwLock;
 
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState};
 use runtime_primitives::traits::{Block as BlockT, NumberFor, AuthorityIdFor, Zero, Header};
 use crate::in_mem::{self, check_genesis_storage};
@@ -268,11 +268,11 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> ClientResult<H::Out> {
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> ClientResult<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		// this is only called when genesis block is imported => shouldn't be performance bottleneck
-		let mut storage: HashMap<Option<Vec<u8>>, StorageMap> = HashMap::new();
+		let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
 		storage.insert(None, top);
 		for (child_key, child_storage) in children {
 			storage.insert(Some(child_key), child_storage);

substrate/core/service/src/chain_spec.rs

@@ -21,7 +21,7 @@ use std::fs::File;
 use std::path::PathBuf;
 use serde_derive::{Serialize, Deserialize};
 use primitives::storage::{StorageKey, StorageData};
-use runtime_primitives::{BuildStorage, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{BuildStorage, StorageOverlay, ChildrenStorageOverlay};
 use serde_json as json;
 use crate::components::RuntimeGenesis;
 use network::Multiaddr;
@@ -65,7 +65,7 @@ impl<G: RuntimeGenesis> GenesisSource<G> {
 }
 
 impl<'a, G: RuntimeGenesis> BuildStorage for &'a ChainSpec<G> {
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String> {
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		match self.genesis.resolve()? {
 			Genesis::Runtime(gc) => gc.build_storage(),
 			Genesis::Raw(map) => Ok((map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), Default::default())),

substrate/core/sr-primitives/src/lib.rs

@@ -89,11 +89,11 @@ use serde_derive::{Serialize, Deserialize};
 
 /// A set of key value pairs for storage.
 #[cfg(feature = "std")]
-pub type StorageMap = HashMap<Vec<u8>, Vec<u8>>;
+pub type StorageOverlay = HashMap<Vec<u8>, Vec<u8>>;
 
 /// A set of key value pairs for children storage;
 #[cfg(feature = "std")]
-pub type ChildrenStorageMap = HashMap<Vec<u8>, StorageMap>;
+pub type ChildrenStorageOverlay = HashMap<Vec<u8>, StorageOverlay>;
 
 /// Complex storage builder stuff.
 #[cfg(feature = "std")]
@@ -107,12 +107,12 @@ pub trait BuildStorage {
 		r
 	}
 	/// Build the storage out of this builder.
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String>;
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String>;
 }
 
 #[cfg(feature = "std")]
-impl BuildStorage for StorageMap {
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String> {
+impl BuildStorage for StorageOverlay {
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		Ok((self, Default::default()))
 	}
 }
@@ -393,9 +393,9 @@ macro_rules! impl_outer_config {
 		}
 		#[cfg(any(feature = "std", test))]
 		impl $crate::BuildStorage for $main {
-			fn build_storage(self) -> ::std::result::Result<($crate::StorageMap, $crate::ChildrenStorageMap), String> {
-				let mut top = $crate::StorageMap::new();
-				let mut children = $crate::ChildrenStorageMap::new();
+			fn build_storage(self) -> ::std::result::Result<($crate::StorageOverlay, $crate::ChildrenStorageOverlay), String> {
+				let mut top = $crate::StorageOverlay::new();
+				let mut children = $crate::ChildrenStorageOverlay::new();
 				$(
 					if let Some(extra) = self.$snake {
 						let (other_top, other_children) = extra.build_storage()?;

substrate/core/test-client/src/lib.rs

@@ -35,7 +35,7 @@ pub use consensus;
 
 use std::sync::Arc;
 use primitives::Blake2Hasher;
-use runtime_primitives::StorageMap;
+use runtime_primitives::StorageOverlay;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT};
 use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
 use keyring::Keyring;
@@ -126,7 +126,7 @@ fn genesis_config(support_changes_trie: bool) -> GenesisConfig {
 	], 1000)
 }
 
-fn genesis_storage(support_changes_trie: bool) -> StorageMap {
+fn genesis_storage(support_changes_trie: bool) -> StorageOverlay {
 	let mut storage = genesis_config(support_changes_trie).genesis_map();
 	let state_root = <<<runtime::Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(storage.clone().into_iter());
 	let block: runtime::Block = client::genesis::construct_genesis_block(state_root);

substrate/srml/consensus/src/lib.rs

@@ -168,7 +168,7 @@ decl_storage! {
 		#[serde(with = "substrate_primitives::bytes")]
 		config(code): Vec<u8>;
 
-		build(|storage: &mut primitives::StorageMap, _: &mut primitives::ChildrenStorageMap, config: &GenesisConfig<T>| {
+		build(|storage: &mut primitives::StorageOverlay, _: &mut primitives::ChildrenStorageOverlay, config: &GenesisConfig<T>| {
 			use codec::{Encode, KeyedVec};
 
 			let auth_count = config.authorities.len() as u32;

substrate/srml/grandpa/src/lib.rs

@@ -136,7 +136,7 @@ decl_storage! {
 	add_extra_genesis {
 		config(authorities): Vec<(T::SessionKey, u64)>;
 
-		build(|storage: &mut primitives::StorageMap, _: &mut primitives::ChildrenStorageMap, config: &GenesisConfig<T>| {
+		build(|storage: &mut primitives::StorageOverlay, _: &mut primitives::ChildrenStorageOverlay, config: &GenesisConfig<T>| {
 			use codec::{Encode, KeyedVec};
 
 			let auth_count = config.authorities.len() as u32;

substrate/srml/indices/src/lib.rs

@@ -100,7 +100,7 @@ decl_storage! {
 	}
 	add_extra_genesis {
 		config(ids): Vec<T::AccountId>;
-		build(|storage: &mut primitives::StorageMap, _: &mut primitives::ChildrenStorageMap, config: &GenesisConfig<T>| {
+		build(|storage: &mut primitives::StorageOverlay, _: &mut primitives::ChildrenStorageOverlay, config: &GenesisConfig<T>| {
 			for i in 0..(config.ids.len() + ENUM_SET_SIZE - 1) / ENUM_SET_SIZE {
 				storage.insert(GenesisConfig::<T>::hash(&<EnumSet<T>>::key_for(T::AccountIndex::sa(i))).to_vec(),
 					config.ids[i * ENUM_SET_SIZE..config.ids.len().min((i + 1) * ENUM_SET_SIZE)].to_owned().encode());

substrate/srml/support/procedural/src/lib.rs

@@ -51,7 +51,7 @@ use proc_macro::TokenStream;
 /// 	}
 ///		add_extra_genesis {
 ///			config(genesis_field): GenesisFieldType;
-///			build(|_: &mut StorageMap, _: &mut ChildrenStorageMap, _: &GenesisConfig<T>| {
+///			build(|_: &mut StorageOverlay, _: &mut ChildrenStorageOverlay, _: &GenesisConfig<T>| {
 ///			})
 ///		}
 /// }

substrate/srml/support/procedural/src/storage/impls.rs

@@ -0,0 +1,428 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use proc_macro2::TokenStream as TokenStream2;
+use syn;
+use quote::quote;
+use crate::storage::transformation::DeclStorageTypeInfos;
+
+pub fn option_unwrap(is_option: bool) -> TokenStream2 {
+	if !is_option {
+		// raw type case
+		quote!( unwrap_or_else )
+	} else {
+		// Option<> type case
+		quote!( or_else )
+	}
+}
+
+pub(crate) struct Impls<'a> {
+	pub scrate: &'a TokenStream2,
+	pub visibility: &'a syn::Visibility,
+	pub traitinstance: &'a syn::Ident,
+	pub traittype: &'a syn::TypeParamBound,
+	pub type_infos: DeclStorageTypeInfos<'a>,
+	pub fielddefault: TokenStream2,
+	pub prefix: String,
+	pub name: &'a syn::Ident,
+}
+
+impl<'a> Impls<'a> {
+	pub fn simple_value(self) -> TokenStream2 {
+		let Self {
+			scrate,
+			visibility,
+			traitinstance,
+			traittype,
+			type_infos,
+			fielddefault,
+			prefix,
+			name,
+		} = self;
+		let DeclStorageTypeInfos { typ, value_type, is_option, .. } = type_infos;
+		let option_simple_1 = option_unwrap(is_option);
+
+		let mutate_impl = if !is_option {
+			quote!{
+				<Self as #scrate::storage::generator::StorageValue<#typ>>::put(&val, storage)
+			}
+		} else {
+			quote!{
+				match val {
+					Some(ref val) => <Self as #scrate::storage::generator::StorageValue<#typ>>::put(&val, storage),
+					None => <Self as #scrate::storage::generator::StorageValue<#typ>>::kill(storage),
+				}
+			}
+		};
+
+		// generator for value
+		quote!{
+
+			#visibility struct #name<#traitinstance: #traittype>(#scrate::storage::generator::PhantomData<#traitinstance>);
+
+			impl<#traitinstance: #traittype> #scrate::storage::generator::StorageValue<#typ> for #name<#traitinstance> {
+				type Query = #value_type;
+
+				/// Get the storage key.
+				fn key() -> &'static [u8] {
+					#prefix.as_bytes()
+				}
+
+				/// Load the value from the provided storage instance.
+				fn get<S: #scrate::GenericStorage>(storage: &S) -> Self::Query {
+					storage.get(<Self as #scrate::storage::generator::StorageValue<#typ>>::key())
+						.#option_simple_1(|| #fielddefault)
+				}
+
+				/// Take a value from storage, removing it afterwards.
+				fn take<S: #scrate::GenericStorage>(storage: &S) -> Self::Query {
+					storage.take(<Self as #scrate::storage::generator::StorageValue<#typ>>::key())
+						.#option_simple_1(|| #fielddefault)
+				}
+
+				/// Mutate the value under a key.
+				fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericStorage>(f: F, storage: &S) -> R {
+					let mut val = <Self as #scrate::storage::generator::StorageValue<#typ>>::get(storage);
+
+					let ret = f(&mut val);
+					#mutate_impl ;
+					ret
+				}
+			}
+
+		}
+	}
+
+	pub fn map(self, kty: &syn::Type) -> TokenStream2 {
+		let Self {
+			scrate,
+			visibility,
+			traitinstance,
+			traittype,
+			type_infos,
+			fielddefault,
+			prefix,
+			name,
+		} = self;
+		let DeclStorageTypeInfos { typ, value_type, is_option, .. } = type_infos;
+		let option_simple_1 = option_unwrap(is_option);
+
+		let mutate_impl = if !is_option {
+			quote!{
+				<Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::insert(key, &val, storage)
+			}
+		} else {
+			quote!{
+				match val {
+					Some(ref val) => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::insert(key, &val, storage),
+					None => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::remove(key, storage),
+				}
+			}
+		};
+		// generator for map
+		quote!{
+			#visibility struct #name<#traitinstance: #traittype>(#scrate::storage::generator::PhantomData<#traitinstance>);
+
+			impl<#traitinstance: #traittype> #scrate::storage::generator::StorageMap<#kty, #typ> for #name<#traitinstance> {
+				type Query = #value_type;
+
+				/// Get the prefix key in storage.
+				fn prefix() -> &'static [u8] {
+					#prefix.as_bytes()
+				}
+
+				/// Get the storage key used to fetch a value corresponding to a specific key.
+				fn key_for(x: &#kty) -> #scrate::rstd::vec::Vec<u8> {
+					let mut key = #prefix.as_bytes().to_vec();
+					#scrate::codec::Encode::encode_to(x, &mut key);
+					key
+				}
+
+				/// Load the value associated with the given key from the map.
+				fn get<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
+					let key = <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::key_for(key);
+					storage.get(&key[..]).#option_simple_1(|| #fielddefault)
+				}
+
+				/// Take the value, reading and removing it.
+				fn take<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
+					let key = <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::key_for(key);
+					storage.take(&key[..]).#option_simple_1(|| #fielddefault)
+				}
+
+				/// Mutate the value under a key
+				fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericStorage>(key: &#kty, f: F, storage: &S) -> R {
+					let mut val = <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::get(key, storage);
+
+					let ret = f(&mut val);
+					#mutate_impl ;
+					ret
+				}
+
+			}
+		}
+	}
+
+	pub fn linked_map(self, kty: &syn::Type) -> TokenStream2 {
+		let Self {
+			scrate,
+			visibility,
+			traitinstance,
+			traittype,
+			type_infos,
+			fielddefault,
+			prefix,
+			name,
+		} = self;
+		let DeclStorageTypeInfos { typ, value_type, is_option, .. } = type_infos;
+		let option_simple_1 = option_unwrap(is_option);
+		// make sure to use different prefix for head and elements.
+		let head_key = format!("head of {}", prefix);
+		let prefix = format!("{}", prefix);
+		let name_lowercase = name.to_string().to_lowercase();
+		let key_for = syn::Ident::new(&format!("key_for_{}", name_lowercase), name.span());
+		let internal_module = syn::Ident::new(&format!("__internal_do_not_use_{}", name_lowercase), name.span());
+		let linkage = syn::Ident::new(&format!("Linkage{}", name), name.span());
+		let borrowing_linkage = syn::Ident::new(&format!("Borrowing{}", linkage), name.span());
+		let enumerator = syn::Ident::new(&format!("Enumerator{}", name), name.span());
+		let put_or_insert = quote! {
+			match linkage {
+				Some(linkage) => storage.put(key_for, &(val, linkage)),
+				None => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::insert(key, &val, storage),
+			}
+		};
+		let mutate_impl = if !type_infos.is_option {
+			put_or_insert
+		} else {
+			quote! {
+				match val {
+					Some(ref val) => #put_or_insert,
+					None => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::remove(key, storage),
+				}
+			}
+		};
+
+		// generator for linked map
+		quote! {
+
+			mod #internal_module {
+				use super::*;
+
+				#[derive(Default, parity_codec_derive::Encode, parity_codec_derive::Decode)]
+				pub struct #linkage {
+					/// Previous element key in storage (None for the first element)
+					previous: Option<#kty>,
+					/// Next element key in storage (None for the last element)
+					next: Option<#kty>,
+				}
+
+				/// A helper struct to avoid unnecessary key cloning.
+				///
+				/// NOTE It has to have exact same parity-codec encoding as #linkage!
+				#[derive(parity_codec_derive::Encode)]
+				struct #borrowing_linkage<'a> {
+					previous: Option<&'a #kty>,
+					next: Option<&'a #kty>,
+				}
+
+				impl #linkage {
+					/// Update linkage when this element is removed.
+					///
+					/// Takes care of updating previous and next elements points
+					/// as well as updates head if the element is first or last.
+					pub fn remove<S: #scrate::GenericStorage>(
+						self,
+						storage: &S,
+					) {
+						let next_key = self.next.as_ref().map(|x| #key_for(x));
+						let prev_key = self.previous.as_ref().map(|x| #key_for(x));
+
+						if let Some(prev_key) = prev_key {
+							// Retrieve previous element and update `next`
+							let mut res = Self::read(storage, &*prev_key)
+								.expect("Linkage is updated in case entry is removed; it always points to existing keys; qed");
+							res.1.next = self.next;
+							storage.put(&*prev_key, &res);
+						} else {
+							// we were first so let's update the head
+							Self::write_head(storage, self.next.as_ref());
+						}
+
+						if let Some(next_key) = next_key {
+							// Update previous of next element
+							let mut res = Self::read(storage, &*next_key)
+								.expect("Linkage is updated in case entry is removed; it always points to existing keys; qed");
+							res.1.previous = self.previous;
+							storage.put(&*next_key, &res);
+						}
+					}
+
+					/// Read the contained data and it's linkage.
+					pub fn read<S: #scrate::GenericStorage>(storage: &S, key: &[u8]) -> Option<(#value_type, #linkage)> {
+						storage.get(key)
+					}
+
+					/// Generate linkage for newly inserted element.
+					///
+					/// Takes care of updating head and previous head's pointer.
+					pub fn insert_new_head<S: #scrate::GenericStorage>(
+						storage: &S,
+						key: &#kty,
+					) -> Self {
+						if let Some(head) = Self::read_head(storage) {
+							// update previous head predecessor
+							{
+								let head_key = #key_for(&head);
+								let (data, linkage) = Self::read(storage, &*head_key).expect(r#"
+									head is set when first element is inserted and unset when last element is removed;
+									if head is Some then it points to existing key; qed
+								"#);
+								storage.put(&*head_key, &(data, #borrowing_linkage {
+									next: linkage.next.as_ref(),
+									previous: Some(key),
+								}));
+							}
+							// update to current head
+							Self::write_head(storage, Some(key));
+							// return linkage with pointer to previous head
+							let mut linkage = Self::default();
+							linkage.next = Some(head);
+							linkage
+						} else {
+							// we are first - update the head and produce empty linkage
+							Self::write_head(storage, Some(key));
+							Self::default()
+						}
+					}
+
+					/// Read current head pointer.
+					pub fn read_head<S: #scrate::GenericStorage>(storage: &S) -> Option<#kty> {
+						storage.get(#head_key.as_bytes())
+					}
+
+					/// Overwrite current head pointer.
+					///
+					/// If `None` is given head is removed from storage.
+					fn write_head<S: #scrate::GenericStorage>(storage: &S, head: Option<&#kty>) {
+						match head {
+							Some(head) => storage.put(#head_key.as_bytes(), head),
+							None => storage.kill(#head_key.as_bytes()),
+						}
+					}
+				}
+
+				pub struct #enumerator<'a, S> {
+					pub storage: &'a S,
+					pub next: Option<#kty>,
+				}
+
+				impl<'a, S: #scrate::GenericStorage> Iterator for #enumerator<'a, S> {
+					type Item = (#kty, #typ);
+
+					fn next(&mut self) -> Option<Self::Item> {
+						let next = self.next.take()?;
+						let key_for = #key_for(&next);
+						let (val, linkage) = #linkage::read(self.storage, &*key_for)
+							.expect("previous/next only contain existing entires; we enumerate using next; entry exists; qed");
+						self.next = linkage.next;
+						Some((next, val))
+					}
+				}
+			}
+
+			fn #key_for(key: &#kty) -> #scrate::rstd::vec::Vec<u8> {
+				let mut key_for = #prefix.as_bytes().to_vec();
+				#scrate::codec::Encode::encode_to(&key, &mut key_for);
+				key_for
+			}
+
+			#visibility struct #name<#traitinstance: #traittype>(#scrate::storage::generator::PhantomData<#traitinstance>);
+
+			impl<#traitinstance: #traittype> #scrate::storage::generator::StorageMap<#kty, #typ> for #name<#traitinstance> {
+				type Query = #value_type;
+
+				/// Get the prefix key in storage.
+				fn prefix() -> &'static [u8] {
+					#prefix.as_bytes()
+				}
+
+				/// Get the storage key used to fetch a value corresponding to a specific key.
+				fn key_for(x: &#kty) -> #scrate::rstd::vec::Vec<u8> {
+					#key_for(x)
+				}
+
+				/// Load the value associated with the given key from the map.
+				fn get<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
+					storage.get(&*#key_for(key)).#option_simple_1(|| #fielddefault)
+				}
+
+				/// Take the value, reading and removing it.
+				fn take<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
+					let res: Option<(#value_type, self::#internal_module::#linkage)> = storage.take(&*#key_for(key));
+					match res {
+						Some((data, linkage)) => {
+							linkage.remove(storage);
+							data
+						},
+						None => #fielddefault
+					}
+				}
+
+				/// Remove the value under a key.
+				fn remove<S: #scrate::GenericStorage>(key: &#kty, storage: &S) {
+					<Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::take(key, storage);
+				}
+
+				/// Store a value to be associated with the given key from the map.
+				fn insert<S: #scrate::GenericStorage>(key: &#kty, val: &#typ, storage: &S) {
+					let key_for = &*#key_for(key);
+					let linkage = match self::#internal_module::#linkage::read(storage, key_for) {
+						// overwrite but reuse existing linkage
+						Some((_data, linkage)) => linkage,
+						// create new linkage
+						None => self::#internal_module::#linkage::insert_new_head(storage, key),
+					};
+					storage.put(key_for, &(*val, linkage))
+				}
+
+				/// Mutate the value under a key
+				fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericStorage>(key: &#kty, f: F, storage: &S) -> R {
+					let key_for = &*#key_for(key);
+					let (mut val, linkage) = self::#internal_module::#linkage::read(storage, key_for)
+						.map(|(data, linkage)| (data, Some(linkage)))
+						.#option_simple_1(|| (#fielddefault, None));
+
+					let ret = f(&mut val);
+					#mutate_impl ;
+					ret
+				}
+			}
+
+			impl<#traitinstance: #traittype> #scrate::storage::generator::EnumerableStorageMap<#kty, #typ> for #name<#traitinstance> {
+				fn head<S: #scrate::GenericStorage>(storage: &S) -> Option<#kty> {
+					self::#internal_module::#linkage::read_head(storage)
+				}
+
+				fn enumerate<'a, S: #scrate::GenericStorage>(storage: &'a S) -> #scrate::storage::generator::Box<dyn Iterator<Item = (#kty, #typ)> + 'a> {
+					#scrate::storage::generator::Box::new(self::#internal_module::#enumerator {
+						next: self::#internal_module::#linkage::read_head(storage),
+						storage,
+					})
+				}
+			}
+		}
+	}
+}

substrate/srml/support/procedural/src/storage/mod.rs

@@ -24,6 +24,8 @@ use srml_support_procedural_tools::{ToTokens, Parse, custom_keyword, custom_keyw
 use syn::{Ident, Token};
 use syn::token::CustomKeyword;
 
+mod impls;
+
 pub mod transformation;
 
 /// Parsing usage only
@@ -116,6 +118,7 @@ struct DeclStorageBuild {
 #[derive(Parse, ToTokens, Debug)]
 enum DeclStorageType {
 	Map(DeclStorageMap),
+	LinkedMap(DeclStorageLinkedMap),
 	Simple(syn::Type),
 }
 
@@ -127,6 +130,14 @@ struct DeclStorageMap {
 	pub value: syn::Type,
 }
 
+#[derive(Parse, ToTokens, Debug)]
+struct DeclStorageLinkedMap {
+	pub map_keyword: ext::CustomToken<LinkedMapKeyword>,
+	pub key: syn::Type,
+	pub ass_keyword: Token![=>],
+	pub value: syn::Type,
+}
+
 #[derive(Parse, ToTokens, Debug)]
 struct DeclStorageDefault {
 	pub equal_token: Token![=],
@@ -141,3 +152,4 @@ custom_keyword_impl!(DeclStorageBuild, "build", "storage build config");
 custom_keyword_impl!(AddExtraGenesis, "add_extra_genesis", "storage extra genesis");
 custom_keyword_impl!(DeclStorageGetter, "get", "storage getter");
 custom_keyword!(MapKeyword, "map", "map as keyword");
+custom_keyword!(LinkedMapKeyword, "linked_map", "linked_map as keyword");

substrate/srml/support/procedural/src/storage/transformation.rs

@@ -188,15 +188,15 @@ fn decl_store_extra_genesis(
 				let ident = &getter.getfn.content;
 				quote!( #ident )
 			};
-			if type_infos.is_simple && ext::has_parametric_type(type_infos.full_type, traitinstance) {
+			if type_infos.kind.is_simple() && ext::has_parametric_type(type_infos.value_type, traitinstance) {
 				is_trait_needed = true;
 				has_trait_field = true;
 			}
-			for t in ext::get_non_bound_serde_derive_types(type_infos.full_type, &traitinstance).into_iter() {
+			for t in ext::get_non_bound_serde_derive_types(type_infos.value_type, &traitinstance).into_iter() {
 				serde_complete_bound.insert(t);
 			}
-			if let Some(kt) = type_infos.map_key {
-				for t in ext::get_non_bound_serde_derive_types(kt, &traitinstance).into_iter() {
+			if let DeclStorageTypeInfosKind::Map { key_type, .. } = type_infos.kind {
+				for t in ext::get_non_bound_serde_derive_types(key_type, &traitinstance).into_iter() {
 					serde_complete_bound.insert(t);
 				}
 			}
@@ -210,8 +210,7 @@ fn decl_store_extra_genesis(
 				} else {
 					quote!( #d )
 				}).unwrap_or_else(|| quote!( Default::default() ));
-					config_field_default.extend(quote!( #ident: #fielddefault, ));
-
+			config_field_default.extend(quote!( #ident: #fielddefault, ));
 		} else {
 			opt_build = build.as_ref().map(|b| &b.expr.content).map(|b| quote!( #b ));
 		}
@@ -219,25 +218,39 @@ fn decl_store_extra_genesis(
 		let typ = type_infos.typ;
 		if let Some(builder) = opt_build {
 			is_trait_needed = true;
-			if type_infos.is_simple {
-				builders.extend(quote!{{
-					use #scrate::codec::Encode;
-					let v = (#builder)(&self);
-					r.insert(Self::hash(
-						<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>>::key()
-						).to_vec(), v.encode());
-				}});
-			} else {
-				let kty = type_infos.map_key.clone().expect("is not simple; qed");
-				builders.extend(quote!{{
-					use #scrate::codec::Encode;
-					let data = (#builder)(&self);
-					for (k, v) in data.into_iter() {
-						let key = <#name<#traitinstance> as #scrate::storage::generator::StorageMap<#kty, #typ>>::key_for(&k);
-						r.insert(Self::hash(&key[..]).to_vec(), v.encode());
-					}
-				}});
-			}
+			let error_message = format!(
+				"Genesis parameters encoding of {} does not match the expected type ({:?}).",
+				name,
+				type_infos.value_type,
+			);
+			builders.extend(match type_infos.kind {
+				DeclStorageTypeInfosKind::Simple => {
+					quote!{{
+						use #scrate::rstd::{cell::RefCell, marker::PhantomData};
+						use #scrate::codec::{Encode, Decode};
+
+						let storage = (RefCell::new(&mut r), PhantomData::<Self>::default());
+						let v = (#builder)(&self);
+						let v = Encode::using_encoded(&v, |mut v| Decode::decode(&mut v))
+							.expect(#error_message);
+						<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>>::put(&v, &storage);
+					}}
+				},
+				DeclStorageTypeInfosKind::Map { key_type, .. } => {
+					quote!{{
+						use #scrate::rstd::{cell::RefCell, marker::PhantomData};
+						use #scrate::codec::{Encode, Decode};
+
+						let storage = (RefCell::new(&mut r), PhantomData::<Self>::default());
+						let data = (#builder)(&self);
+						for (k, v) in data.into_iter() {
+							let v = Encode::using_encoded(&v, |mut v| Decode::decode(&mut v))
+								.expect(#error_message);
+							<#name<#traitinstance> as #scrate::storage::generator::StorageMap<#key_type, #typ>>::insert(&k, &v, &storage);
+						}
+					}}
+				},
+			});
 		}
 
 	}
@@ -367,9 +380,9 @@ fn decl_store_extra_genesis(
 			#[cfg(feature = "std")]
 			impl#fparam #scrate::runtime_primitives::BuildStorage for GenesisConfig#sparam {
 
-				fn build_storage(self) -> ::std::result::Result<(#scrate::runtime_primitives::StorageMap, #scrate::runtime_primitives::ChildrenStorageMap), String> {
-					let mut r: #scrate::runtime_primitives::StorageMap = Default::default();
-					let mut c: #scrate::runtime_primitives::ChildrenStorageMap = Default::default();
+				fn build_storage(self) -> ::std::result::Result<(#scrate::runtime_primitives::StorageOverlay, #scrate::runtime_primitives::ChildrenStorageOverlay), String> {
+					let mut r: #scrate::runtime_primitives::StorageOverlay = Default::default();
+					let mut c: #scrate::runtime_primitives::ChildrenStorageOverlay = Default::default();
 
 					#builders
 
@@ -403,128 +416,29 @@ fn decl_storage_items(
 		} = sline;
 
 		let type_infos = get_type_infos(storage_type);
-		let gettype = type_infos.full_type;
-		let fielddefault = default_value.inner.as_ref().map(|d| &d.expr).map(|d| quote!( #d ))
-			.unwrap_or_else(|| quote!{ Default::default() });
-
-		let typ = type_infos.typ;
-
-		let option_simple_1 = if !type_infos.is_option {
-			// raw type case
-			quote!( unwrap_or_else )
-		} else {
-			// Option<> type case
-			quote!( or_else )
+		let kind = type_infos.kind.clone();
+		let i = impls::Impls {
+			scrate,
+			visibility,
+			traitinstance,
+			traittype,
+			type_infos,
+			fielddefault: default_value.inner.as_ref().map(|d| &d.expr).map(|d| quote!( #d ))
+				.unwrap_or_else(|| quote!{ Default::default() }),
+			prefix: format!("{} {}", cratename, name),
+			name,
 		};
-		let implementation = if type_infos.is_simple {
-			let mutate_impl = if !type_infos.is_option {
-				quote!{
-					<Self as #scrate::storage::generator::StorageValue<#typ>>::put(&val, storage)
-				}
-			} else {
-				quote!{
-					match val {
-						Some(ref val) => <Self as #scrate::storage::generator::StorageValue<#typ>>::put(&val, storage),
-						None => <Self as #scrate::storage::generator::StorageValue<#typ>>::kill(storage),
-					}
-				}
-			};
 
-			let key_string = cratename.to_string() + " " + &name.to_string();
-			// generator for value
-			quote!{
-
-				#visibility struct #name<#traitinstance: #traittype>(#scrate::storage::generator::PhantomData<#traitinstance>);
-
-				impl<#traitinstance: #traittype> #scrate::storage::generator::StorageValue<#typ> for #name<#traitinstance> {
-					type Query = #gettype;
-
-					/// Get the storage key.
-					fn key() -> &'static [u8] {
-						#key_string.as_bytes()
-					}
-
-					/// Load the value from the provided storage instance.
-					fn get<S: #scrate::GenericStorage>(storage: &S) -> Self::Query {
-						storage.get(<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>>::key())
-							.#option_simple_1(|| #fielddefault)
-					}
-
-					/// Take a value from storage, removing it afterwards.
-					fn take<S: #scrate::GenericStorage>(storage: &S) -> Self::Query {
-						storage.take(<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>>::key())
-							.#option_simple_1(|| #fielddefault)
-					}
-
-					/// Mutate the value under a key.
-					fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericStorage>(f: F, storage: &S) -> R {
-						let mut val = <Self as #scrate::storage::generator::StorageValue<#typ>>::get(storage);
-
-						let ret = f(&mut val);
-						#mutate_impl ;
-						ret
-					}
-				}
-
-			}
-		} else {
-			let kty = type_infos.map_key.expect("is not simple; qed");
-			let mutate_impl = if !type_infos.is_option {
-				quote!{
-					<Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::insert(key, &val, storage)
-				}
-			} else {
-				quote!{
-					match val {
-						Some(ref val) => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::insert(key, &val, storage),
-						None => <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::remove(key, storage),
-					}
-				}
-			};
-			let prefix_string = cratename.to_string() + " " + &name.to_string();
-			// generator for map
-			quote!{
-				#visibility struct #name<#traitinstance: #traittype>(#scrate::storage::generator::PhantomData<#traitinstance>);
-
-				impl<#traitinstance: #traittype> #scrate::storage::generator::StorageMap<#kty, #typ> for #name<#traitinstance> {
-					type Query = #gettype;
-
-					/// Get the prefix key in storage.
-					fn prefix() -> &'static [u8] {
-						#prefix_string.as_bytes()
-					}
-
-					/// Get the storage key used to fetch a value corresponding to a specific key.
-					fn key_for(x: &#kty) -> #scrate::rstd::vec::Vec<u8> {
-						let mut key = <#name<#traitinstance> as #scrate::storage::generator::StorageMap<#kty, #typ>>::prefix().to_vec();
-						#scrate::codec::Encode::encode_to(x, &mut key);
-						key
-					}
-
-					/// Load the value associated with the given key from the map.
-					fn get<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
-						let key = <#name<#traitinstance> as #scrate::storage::generator::StorageMap<#kty, #typ>>::key_for(key);
-						storage.get(&key[..]).#option_simple_1(|| #fielddefault)
-					}
-
-					/// Take the value, reading and removing it.
-					fn take<S: #scrate::GenericStorage>(key: &#kty, storage: &S) -> Self::Query {
-						let key = <#name<#traitinstance> as #scrate::storage::generator::StorageMap<#kty, #typ>>::key_for(key);
-						storage.take(&key[..]).#option_simple_1(|| #fielddefault)
-					}
-
-					/// Mutate the value under a key
-					fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: #scrate::GenericStorage>(key: &#kty, f: F, storage: &S) -> R {
-						let mut val = <Self as #scrate::storage::generator::StorageMap<#kty, #typ>>::take(key, storage);
-
-						let ret = f(&mut val);
-						#mutate_impl ;
-						ret
-					}
-
-				}
-
-			}
+		let implementation = match kind {
+			DeclStorageTypeInfosKind::Simple => {
+				i.simple_value()
+			},
+			DeclStorageTypeInfosKind::Map { key_type, is_linked: false } => {
+				i.map(key_type)
+			},
+			DeclStorageTypeInfosKind::Map { key_type, is_linked: true } => {
+				i.linked_map(key_type)
+			},
 		};
 		impls.extend(implementation)
 	}
@@ -571,21 +485,22 @@ fn impl_store_fns(
 			let get_fn = &getter.getfn.content;
 
 			let type_infos = get_type_infos(storage_type);
-			let gettype = type_infos.full_type;
+			let value_type = type_infos.value_type;
 
 			let typ = type_infos.typ;
-			let item = if type_infos.is_simple {
-				quote!{
-					pub fn #get_fn() -> #gettype {
-						<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>> :: get(&#scrate::storage::RuntimeStorage)
+			let item = match type_infos.kind {
+				DeclStorageTypeInfosKind::Simple => {
+					quote!{
+						pub fn #get_fn() -> #value_type {
+							<#name<#traitinstance> as #scrate::storage::generator::StorageValue<#typ>> :: get(&#scrate::storage::RuntimeStorage)
+						}
 					}
-				}
-			} else {
-				let kty = type_infos.map_key.expect("is not simple; qed");
-				// map
-				quote!{
-					pub fn #get_fn<K: #scrate::storage::generator::Borrow<#kty>>(key: K) -> #gettype {
-						<#name<#traitinstance> as #scrate::storage::generator::StorageMap<#kty, #typ>> :: get(key.borrow(), &#scrate::storage::RuntimeStorage)
+				},
+				DeclStorageTypeInfosKind::Map { key_type, .. } => {
+					quote!{
+						pub fn #get_fn<K: #scrate::storage::generator::Borrow<#key_type>>(key: K) -> #value_type {
+							<#name<#traitinstance> as #scrate::storage::generator::StorageMap<#key_type, #typ>> :: get(key.borrow(), &#scrate::storage::RuntimeStorage)
+						}
 					}
 				}
 			};
@@ -614,26 +529,27 @@ fn store_functions_to_metadata (
 		} = sline;
 
 		let type_infos = get_type_infos(storage_type);
-		let gettype = type_infos.full_type;
+		let value_type = type_infos.value_type;
 
 		let typ = type_infos.typ;
-		let stype = if type_infos.is_simple {
-			let styp = clean_type_string(&typ.to_string());
-			quote!{
-				#scrate::storage::generator::StorageFunctionType::Plain(
-					#scrate::storage::generator::DecodeDifferent::Encode(#styp),
-				)
-			}
-		} else {
-			let kty = type_infos.map_key.expect("is not simple; qed");
-			let kty = clean_type_string(&quote!(#kty).to_string());
-			let styp = clean_type_string(&typ.to_string());
-			quote!{
-				#scrate::storage::generator::StorageFunctionType::Map {
-					key: #scrate::storage::generator::DecodeDifferent::Encode(#kty),
-					value: #scrate::storage::generator::DecodeDifferent::Encode(#styp),
+		let styp = clean_type_string(&typ.to_string());
+		let stype = match type_infos.kind {
+			DeclStorageTypeInfosKind::Simple => {
+				quote!{
+					#scrate::storage::generator::StorageFunctionType::Plain(
+						#scrate::storage::generator::DecodeDifferent::Encode(#styp),
+					)
 				}
-			}
+			},
+			DeclStorageTypeInfosKind::Map { key_type, .. } => {
+				let kty = clean_type_string(&quote!(#key_type).to_string());
+				quote!{
+					#scrate::storage::generator::StorageFunctionType::Map {
+						key: #scrate::storage::generator::DecodeDifferent::Encode(#kty),
+						value: #scrate::storage::generator::DecodeDifferent::Encode(#styp),
+					}
+				}
+			},
 		};
 		let modifier = if type_infos.is_option {
 			quote!{
@@ -688,8 +604,8 @@ fn store_functions_to_metadata (
 				fn default_byte(&self) -> #scrate::rstd::vec::Vec<u8> {
 					use #scrate::codec::Encode;
 					#cache_name.get_or_init(|| {
-						let def_val: #gettype = #default;
-						<#gettype as Encode>::encode(&def_val)
+						let def_val: #value_type = #default;
+						<#value_type as Encode>::encode(&def_val)
 					}).clone()
 				}
 			}
@@ -697,8 +613,8 @@ fn store_functions_to_metadata (
 			impl<#traitinstance: #traittype> #scrate::storage::generator::DefaultByte for #struct_name<#traitinstance> {
 				fn default_byte(&self) -> #scrate::rstd::vec::Vec<u8> {
 					use #scrate::codec::Encode;
-					let def_val: #gettype = #default;
-					<#gettype as Encode>::encode(&def_val)
+					let def_val: #value_type = #default;
+					<#value_type as Encode>::encode(&def_val)
 				}
 			}
 		};
@@ -714,26 +630,54 @@ fn store_functions_to_metadata (
 }
 
 
-struct DeclStorageTypeInfos<'a> {
-	pub is_simple: bool,
-	pub full_type: &'a syn::Type,
+#[derive(Debug, Clone)]
+pub(crate) struct DeclStorageTypeInfos<'a> {
 	pub is_option: bool,
 	pub typ: TokenStream2,
-	pub map_key: Option<&'a syn::Type>,
+	pub value_type: &'a syn::Type,
+	kind: DeclStorageTypeInfosKind<'a>,
+}
+
+#[derive(Debug, Clone)]
+enum DeclStorageTypeInfosKind<'a> {
+	Simple,
+	Map {
+		key_type: &'a syn::Type,
+		is_linked: bool,
+	},
+}
+
+impl<'a> DeclStorageTypeInfosKind<'a> {
+	fn is_simple(&self) -> bool {
+		match *self {
+			DeclStorageTypeInfosKind::Simple => true,
+			_ => false,
+		}
+	}
 }
 
 fn get_type_infos(storage_type: &DeclStorageType) -> DeclStorageTypeInfos {
-	let (is_simple, extracted_type, map_key, full_type) = match storage_type {
-		DeclStorageType::Simple(ref st) => (true, ext::extract_type_option(st), None, st),
-		DeclStorageType::Map(ref map) => (false, ext::extract_type_option(&map.value), Some(&map.key), &map.value),
+	let (value_type, kind) = match storage_type {
+		DeclStorageType::Simple(ref st) => (st, DeclStorageTypeInfosKind::Simple),
+		DeclStorageType::Map(ref map) => (&map.value, DeclStorageTypeInfosKind::Map {
+			key_type: &map.key,
+			is_linked: false,
+		}),
+		DeclStorageType::LinkedMap(ref map) => (&map.value, DeclStorageTypeInfosKind::Map {
+			key_type: &map.key,
+			is_linked: true,
+		}),
 	};
+
+	let extracted_type = ext::extract_type_option(value_type);
 	let is_option = extracted_type.is_some();
-	let typ = extracted_type.unwrap_or(quote!( #full_type ));
+	let typ = extracted_type.unwrap_or(quote!( #value_type ));
+
 	DeclStorageTypeInfos {
-		is_simple,
-		full_type,
 		is_option,
 		typ,
-		map_key,
+		value_type,
+		kind,
 	}
+
 }

substrate/srml/support/src/lib.rs

@@ -54,7 +54,7 @@ pub mod inherent;
 mod double_map;
 pub mod traits;
 
-pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap};
+pub use self::storage::{StorageVec, StorageList, StorageValue, StorageMap, EnumerableStorageMap};
 pub use self::hashable::Hashable;
 pub use self::dispatch::{Parameter, Dispatchable, Callable, IsSubType};
 pub use self::double_map::StorageDoubleMap;
@@ -132,3 +132,125 @@ macro_rules! for_each_tuple {
 		for_each_tuple! { @IMPL $m !! $($t,)* }
 	}
 }
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use runtime_io::{with_externalities, Blake2Hasher};
+	use runtime_primitives::BuildStorage;
+
+	pub trait Trait {
+		type BlockNumber;
+		type Origin;
+	}
+
+	mod module {
+		#![allow(dead_code)]
+
+		use super::Trait;
+
+		decl_module! {
+			pub struct Module<T: Trait> for enum Call where origin: T::Origin {
+
+			}
+		}
+	}
+	use self::module::Module;
+
+	decl_storage! {
+		trait Store for Module<T: Trait> as Example {
+			pub Data get(data) build(|_| vec![(15u32, 42u64)]): linked_map u32 => u64;
+		}
+	}
+
+	struct Test;
+	impl Trait for Test {
+		type BlockNumber = u32;
+		type Origin = u32;
+	}
+
+	fn new_test_ext() -> runtime_io::TestExternalities<Blake2Hasher> {
+		GenesisConfig::<Test>::default().build_storage().unwrap().0.into()
+	}
+
+	type Map = Data<Test>;
+
+	#[test]
+	fn basic_insert_remove_should_work() {
+		with_externalities(&mut new_test_ext(), || {
+			// initialised during genesis
+			assert_eq!(Map::get(&15u32), 42u64);
+
+			// get / insert / take
+			let key = 17u32;
+			assert_eq!(Map::get(&key), 0u64);
+			Map::insert(key, 4u64);
+			assert_eq!(Map::get(&key), 4u64);
+			assert_eq!(Map::take(&key), 4u64);
+			assert_eq!(Map::get(&key), 0u64);
+
+			// mutate
+			Map::mutate(&key, |val| {
+				*val = 15;
+			});
+			assert_eq!(Map::get(&key), 15u64);
+
+			// remove
+			Map::remove(&key);
+			assert_eq!(Map::get(&key), 0u64);
+		});
+	}
+
+	#[test]
+	fn enumeration_and_head_should_work() {
+		with_externalities(&mut new_test_ext(), || {
+			assert_eq!(Map::head(), Some(15));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(15, 42)]);
+			// insert / remove
+			let key = 17u32;
+			Map::insert(key, 4u64);
+			assert_eq!(Map::head(), Some(key));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 4), (15, 42)]);
+			assert_eq!(Map::take(&15), 42u64);
+			assert_eq!(Map::take(&key), 4u64);
+			assert_eq!(Map::head(), None);
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![]);
+
+			// Add couple of more elements
+			Map::insert(key, 42u64);
+			assert_eq!(Map::head(), Some(key));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 42)]);
+			Map::insert(key + 1, 43u64);
+			assert_eq!(Map::head(), Some(key + 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key + 1, 43), (key, 42)]);
+
+			// mutate
+			let key = key + 2;
+			Map::mutate(&key, |val| {
+				*val = 15;
+			});
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 15), (key - 1, 43), (key - 2, 42)]);
+			assert_eq!(Map::head(), Some(key));
+			Map::mutate(&key, |val| {
+				*val = 17;
+			});
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key, 17), (key - 1, 43), (key - 2, 42)]);
+
+			// remove first
+			Map::remove(&key);
+			assert_eq!(Map::head(), Some(key - 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key - 1, 43), (key - 2, 42)]);
+
+			// remove last from the list
+			Map::remove(&(key - 2));
+			assert_eq!(Map::head(), Some(key - 1));
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![(key - 1, 43)]);
+
+			// remove the last element
+			Map::remove(&(key - 1));
+			assert_eq!(Map::head(), None);
+			assert_eq!(Map::enumerate().collect::<Vec<_>>(), vec![]);
+		});
+	}
+
+}

substrate/srml/support/src/storage/generator.rs

@@ -52,6 +52,8 @@ use crate::rstd::vec::Vec;
 pub use crate::rstd::borrow::Borrow;
 #[doc(hidden)]
 pub use crate::rstd::marker::PhantomData;
+#[doc(hidden)]
+pub use crate::rstd::boxed::Box;
 
 pub use srml_metadata::{
 	DecodeDifferent, StorageMetadata, StorageFunctionMetadata,
@@ -65,34 +67,55 @@ pub trait Storage {
 	fn exists(&self, key: &[u8]) -> bool;
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
-	fn get<T: codec::Codec>(&self, key: &[u8]) -> Option<T>;
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T>;
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect. Will panic if
 	/// it's not there.
-	fn require<T: codec::Codec>(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") }
+	fn require<T: codec::Decode>(&self, key: &[u8]) -> T { self.get(key).expect("Required values must be in storage") }
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect. The type's
 	/// default is returned if it's not there.
-	fn get_or_default<T: codec::Codec + Default>(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() }
+	fn get_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.get(key).unwrap_or_default() }
 
 	/// Put a value in under a key.
-	fn put<T: codec::Codec>(&self, key: &[u8], val: &T);
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T);
 
 	/// Remove the bytes of a key from storage.
 	fn kill(&self, key: &[u8]);
 
 	/// Take a value from storage, deleting it after reading.
-	fn take<T: codec::Codec>(&self, key: &[u8]) -> Option<T> {
+	fn take<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
 		let value = self.get(key);
 		self.kill(key);
 		value
 	}
 
 	/// Take a value from storage, deleting it after reading.
-	fn take_or_panic<T: codec::Codec>(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") }
+	fn take_or_panic<T: codec::Decode>(&self, key: &[u8]) -> T { self.take(key).expect("Required values must be in storage") }
 
 	/// Take a value from storage, deleting it after reading.
-	fn take_or_default<T: codec::Codec + Default>(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() }
+	fn take_or_default<T: codec::Decode + Default>(&self, key: &[u8]) -> T { self.take(key).unwrap_or_default() }
+}
+
+// We use a construct like this during when genesis storage is being built.
+#[cfg(feature = "std")]
+impl<S: sr_primitives::BuildStorage> Storage for (crate::rstd::cell::RefCell<&mut sr_primitives::StorageOverlay>, PhantomData<S>) {
+	fn exists(&self, key: &[u8]) -> bool {
+		self.0.borrow().contains_key(S::hash(key).as_ref())
+	}
+
+	fn get<T: codec::Decode>(&self, key: &[u8]) -> Option<T> {
+		self.0.borrow().get(S::hash(key).as_ref())
+			.map(|x| codec::Decode::decode(&mut x.as_slice()).expect("Unable to decode expected type."))
+	}
+
+	fn put<T: codec::Encode>(&self, key: &[u8], val: &T) {
+		self.0.borrow_mut().insert(S::hash(key).to_vec(), codec::Encode::encode(val));
+	}
+
+	fn kill(&self, key: &[u8]) {
+		self.0.borrow_mut().remove(S::hash(key).as_ref());
+	}
 }
 
 /// A strongly-typed value kept in storage.
@@ -194,6 +217,15 @@ pub trait StorageMap<K: codec::Codec, V: codec::Codec> {
 	fn mutate<R, F: FnOnce(&mut Self::Query) -> R, S: Storage>(key: &K, f: F, storage: &S) -> R;
 }
 
+/// A `StorageMap` with enumerable entries.
+pub trait EnumerableStorageMap<K: codec::Codec, V: codec::Codec>: StorageMap<K, V> {
+	/// Return current head element.
+	fn head<S: Storage>(storage: &S) -> Option<K>;
+
+	/// Enumerate all elements in the map.
+	fn enumerate<'a, S: Storage>(storage: &'a S) -> Box<dyn Iterator<Item = (K, V)> + 'a>;
+}
+
 // FIXME #1466 Remove this in favour of `decl_storage` macro.
 /// Declares strongly-typed wrappers around codec-compatible types in storage.
 #[macro_export]
@@ -523,7 +555,7 @@ macro_rules! __handle_wrap_internal {
 mod tests {
 	use std::collections::HashMap;
 	use std::cell::RefCell;
-	use codec::Codec;
+	use codec::{Decode, Encode};
 	use super::*;
 	use crate::rstd::marker::PhantomData;
 
@@ -532,11 +564,11 @@ mod tests {
 			self.borrow_mut().get(key).is_some()
 		}
 
-		fn get<T: Codec>(&self, key: &[u8]) -> Option<T> {
+		fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
 			self.borrow_mut().get(key).map(|v| T::decode(&mut &v[..]).unwrap())
 		}
 
-		fn put<T: Codec>(&self, key: &[u8], val: &T) {
+		fn put<T: Encode>(&self, key: &[u8], val: &T) {
 			self.borrow_mut().insert(key.to_owned(), val.encode());
 		}
 

substrate/srml/support/src/storage/mod.rs

@@ -19,7 +19,7 @@
 use crate::rstd::prelude::*;
 use crate::rstd::borrow::Borrow;
 use runtime_io::{self, twox_128};
-use crate::codec::{Codec, Decode, KeyedVec, Input};
+use crate::codec::{Codec, Encode, Decode, KeyedVec, Input};
 
 #[macro_use]
 pub mod generator;
@@ -39,7 +39,7 @@ impl<'a> Input for IncrementalInput<'a> {
 }
 
  /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
-pub fn get<T: Codec + Sized>(key: &[u8]) -> Option<T> {
+pub fn get<T: Decode + Sized>(key: &[u8]) -> Option<T> {
 	let key = twox_128(key);
 	runtime_io::read_storage(&key[..], &mut [0; 0][..], 0).map(|_| {
 		let mut input = IncrementalInput {
@@ -52,29 +52,29 @@ pub fn get<T: Codec + Sized>(key: &[u8]) -> Option<T> {
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
 /// explicit entry.
-pub fn get_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
+pub fn get_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
 	get(key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry.
-pub fn get_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
+pub fn get_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
 	get(key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry.
-pub fn get_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
 	get(key).unwrap_or_else(default_value)
 }
 
 /// Put `value` in storage under `key`.
-pub fn put<T: Codec>(key: &[u8], value: &T) {
+pub fn put<T: Encode>(key: &[u8], value: &T) {
 	value.using_encoded(|slice| runtime_io::set_storage(&twox_128(key)[..], slice));
 }
 
 /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
-pub fn take<T: Codec + Sized>(key: &[u8]) -> Option<T> {
+pub fn take<T: Decode + Sized>(key: &[u8]) -> Option<T> {
 	let r = get(key);
 	if r.is_some() {
 		kill(key);
@@ -84,19 +84,19 @@ pub fn take<T: Codec + Sized>(key: &[u8]) -> Option<T> {
 
 /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
 /// the default for its type.
-pub fn take_or_default<T: Codec + Sized + Default>(key: &[u8]) -> T {
+pub fn take_or_default<T: Decode + Sized + Default>(key: &[u8]) -> T {
 	take(key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or<T: Codec + Sized>(key: &[u8], default_value: T) -> T {
+pub fn take_or<T: Decode + Sized>(key: &[u8], default_value: T) -> T {
 	take(key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or_else<T: Codec + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
+pub fn take_or_else<T: Decode + Sized, F: FnOnce() -> T>(key: &[u8], default_value: F) -> T {
 	take(key).unwrap_or_else(default_value)
 }
 
@@ -129,12 +129,12 @@ impl crate::GenericStorage for RuntimeStorage {
 	}
 
 	/// Load the bytes of a key from storage. Can panic if the type is incorrect.
-	fn get<T: Codec>(&self, key: &[u8]) -> Option<T> {
+	fn get<T: Decode>(&self, key: &[u8]) -> Option<T> {
 		super::storage::get(key)
 	}
 
 	/// Put a value in under a key.
-	fn put<T: Codec>(&self, key: &[u8], val: &T) {
+	fn put<T: Encode>(&self, key: &[u8], val: &T) {
 		super::storage::put(key, val)
 	}
 
@@ -144,7 +144,7 @@ impl crate::GenericStorage for RuntimeStorage {
 	}
 
 	/// Take a value from storage, deleting it after reading.
-	fn take<T: Codec>(&self, key: &[u8]) -> Option<T> {
+	fn take<T: Decode>(&self, key: &[u8]) -> Option<T> {
 		super::storage::take(key)
 	}
 }
@@ -336,6 +336,28 @@ impl<K: Codec, V: Codec, U> StorageMap<K, V> for U where U: generator::StorageMa
 	}
 }
 
+/// A storage map that can be enumerated.
+///
+/// Note that type is primarily useful for off-chain computations.
+/// Runtime implementors should avoid enumerating storage entries.
+pub trait EnumerableStorageMap<K: Codec, V: Codec>: StorageMap<K, V> {
+	/// Return current head element.
+	fn head() -> Option<K>;
+
+	/// Enumerate all elements in the map.
+	fn enumerate() -> Box<dyn Iterator<Item = (K, V)>>;
+}
+
+impl<K: Codec, V: Codec, U> EnumerableStorageMap<K, V> for U where U: generator::EnumerableStorageMap<K, V> {
+	fn head() -> Option<K> {
+		<U as generator::EnumerableStorageMap<K, V>>::head(&RuntimeStorage)
+	}
+
+	fn enumerate() -> Box<dyn Iterator<Item = (K, V)>> {
+		<U as generator::EnumerableStorageMap<K, V>>::enumerate(&RuntimeStorage)
+	}
+}
+
 /// A trait to conveniently store a vector of storable data.
 pub trait StorageVec {
 	type Item: Default + Sized + Codec;

substrate/srml/system/src/lib.rs

@@ -214,7 +214,7 @@ decl_storage! {
 	add_extra_genesis {
 		config(changes_trie_config): Option<ChangesTrieConfiguration>;
 
-		build(|storage: &mut primitives::StorageMap, _: &mut primitives::ChildrenStorageMap, config: &GenesisConfig<T>| {
+		build(|storage: &mut primitives::StorageOverlay, _: &mut primitives::ChildrenStorageOverlay, config: &GenesisConfig<T>| {
 			use parity_codec::Encode;
 
 			storage.insert(well_known_keys::EXTRINSIC_INDEX.to_vec(), 0u32.encode());