EnumerableStorageMap (#1763)

* Refactor decl_storage a bit to allow easier impl of linked map.

* A bunch of refactorings for storage generation.

- Rename StorageMap and ChildrenStorageMap to avoid confusion with generator::StorageMap.
- Separate implementation from the procedural macro code to clean it up.
- Make sure that genesis is initialised using the `StorageValue/StorageMap`
  generated implementations instead of going RAW.

* WiP: Writing test.

* Basic implementation.

* Implement enumeration.

* Fix non-std issues.

* fix warning

* Fix test-client.

* Address review grumbles - part 1

* Avoid cloning the key, relax Storage requirements.

* Rebuild runtime.

* Remove dangling todo.

substrate/core/client/src/backend.rs

@@ -18,7 +18,7 @@
 
 use crate::error;
 use primitives::ChangesTrieConfiguration;
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{AuthorityIdFor, Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
 use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
@@ -79,7 +79,7 @@ pub trait BlockImportOperation<Block, H> where
 	/// Inject storage data into the database.
 	fn update_db_storage(&mut self, update: <Self::State as StateBackend<H>>::Transaction) -> error::Result<()>;
 	/// Inject storage data into the database replacing any existing data.
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out>;
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out>;
 	/// Set top level storage changes.
 	fn update_storage(&mut self, update: Vec<(Vec<u8>, Option<Vec<u8>>)>) -> error::Result<()>;
 	/// Inject changes trie data into the database.

substrate/core/client/src/in_mem.rs

@@ -26,7 +26,7 @@ use primitives::{ChangesTrieConfiguration, storage::well_known_keys};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
 	NumberFor, As, Digest, DigestItem, AuthorityIdFor};
-use runtime_primitives::{Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay};
 use crate::blockchain::{self, BlockStatus, HeaderBackend};
 use state_machine::backend::{Backend as StateBackend, InMemory, Consolidate};
 use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId};
@@ -490,7 +490,7 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, mut top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out> {
+	fn reset_storage(&mut self, mut top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		let mut transaction: Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> = Default::default();
@@ -753,7 +753,7 @@ pub fn cache_authorities_at<Block: BlockT>(
 }
 
 /// Check that genesis storage is valid.
-pub fn check_genesis_storage(top: &StorageMap, children: &ChildrenStorageMap) -> error::Result<()> {
+pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOverlay) -> error::Result<()> {
 	if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 		return Err(error::ErrorKind::GenesisInvalid.into());
 	}

substrate/core/client/src/light/backend.rs

@@ -22,7 +22,7 @@ use std::sync::{Arc, Weak};
 use futures::{Future, IntoFuture};
 use parking_lot::RwLock;
 
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState};
 use runtime_primitives::traits::{Block as BlockT, NumberFor, AuthorityIdFor, Zero, Header};
 use crate::in_mem::{self, check_genesis_storage};
@@ -268,11 +268,11 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> ClientResult<H::Out> {
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> ClientResult<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		// this is only called when genesis block is imported => shouldn't be performance bottleneck
-		let mut storage: HashMap<Option<Vec<u8>>, StorageMap> = HashMap::new();
+		let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
 		storage.insert(None, top);
 		for (child_key, child_storage) in children {
 			storage.insert(Some(child_key), child_storage);