EnumerableStorageMap (#1763)

* Refactor decl_storage a bit to allow easier impl of linked map.

* A bunch of refactorings for storage generation.

- Rename StorageMap and ChildrenStorageMap to avoid confusion with generator::StorageMap.
- Separate implementation from the procedural macro code to clean it up.
- Make sure that genesis is initialised using the `StorageValue/StorageMap`
  generated implementations instead of going RAW.

* WiP: Writing test.

* Basic implementation.

* Implement enumeration.

* Fix non-std issues.

* fix warning

* Fix test-client.

* Address review grumbles - part 1

* Avoid cloning the key, relax Storage requirements.

* Rebuild runtime.

* Remove dangling todo.

substrate/core/client/db/src/lib.rs

@@ -44,7 +44,7 @@ use trie::MemoryDB;
 use parking_lot::RwLock;
 use primitives::{H256, Blake2Hasher, ChangesTrieConfiguration, convert_hash};
 use primitives::storage::well_known_keys;
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero, Digest, DigestItem, AuthorityIdFor};
 use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
@@ -307,7 +307,7 @@ where Block: BlockT<Hash=H256>,
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, mut top: StorageMap, children: ChildrenStorageMap) -> Result<H256, client::error::Error> {
+	fn reset_storage(&mut self, mut top: StorageOverlay, children: ChildrenStorageOverlay) -> Result<H256, client::error::Error> {
 
 		if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 			return Err(client::error::ErrorKind::GenesisInvalid.into());

substrate/core/client/src/backend.rs

@@ -18,7 +18,7 @@
 
 use crate::error;
 use primitives::ChangesTrieConfiguration;
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{AuthorityIdFor, Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
 use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
@@ -79,7 +79,7 @@ pub trait BlockImportOperation<Block, H> where
 	/// Inject storage data into the database.
 	fn update_db_storage(&mut self, update: <Self::State as StateBackend<H>>::Transaction) -> error::Result<()>;
 	/// Inject storage data into the database replacing any existing data.
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out>;
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out>;
 	/// Set top level storage changes.
 	fn update_storage(&mut self, update: Vec<(Vec<u8>, Option<Vec<u8>>)>) -> error::Result<()>;
 	/// Inject changes trie data into the database.

substrate/core/client/src/in_mem.rs

@@ -26,7 +26,7 @@ use primitives::{ChangesTrieConfiguration, storage::well_known_keys};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
 	NumberFor, As, Digest, DigestItem, AuthorityIdFor};
-use runtime_primitives::{Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay};
 use crate::blockchain::{self, BlockStatus, HeaderBackend};
 use state_machine::backend::{Backend as StateBackend, InMemory, Consolidate};
 use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId};
@@ -490,7 +490,7 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, mut top: StorageMap, children: ChildrenStorageMap) -> error::Result<H::Out> {
+	fn reset_storage(&mut self, mut top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		let mut transaction: Vec<(Option<Vec<u8>>, Vec<u8>, Option<Vec<u8>>)> = Default::default();
@@ -753,7 +753,7 @@ pub fn cache_authorities_at<Block: BlockT>(
 }
 
 /// Check that genesis storage is valid.
-pub fn check_genesis_storage(top: &StorageMap, children: &ChildrenStorageMap) -> error::Result<()> {
+pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOverlay) -> error::Result<()> {
 	if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
 		return Err(error::ErrorKind::GenesisInvalid.into());
 	}

substrate/core/client/src/light/backend.rs

@@ -22,7 +22,7 @@ use std::sync::{Arc, Weak};
 use futures::{Future, IntoFuture};
 use parking_lot::RwLock;
 
-use runtime_primitives::{generic::BlockId, Justification, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
 use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState};
 use runtime_primitives::traits::{Block as BlockT, NumberFor, AuthorityIdFor, Zero, Header};
 use crate::in_mem::{self, check_genesis_storage};
@@ -268,11 +268,11 @@ where
 		Ok(())
 	}
 
-	fn reset_storage(&mut self, top: StorageMap, children: ChildrenStorageMap) -> ClientResult<H::Out> {
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> ClientResult<H::Out> {
 		check_genesis_storage(&top, &children)?;
 
 		// this is only called when genesis block is imported => shouldn't be performance bottleneck
-		let mut storage: HashMap<Option<Vec<u8>>, StorageMap> = HashMap::new();
+		let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
 		storage.insert(None, top);
 		for (child_key, child_storage) in children {
 			storage.insert(Some(child_key), child_storage);

substrate/core/service/src/chain_spec.rs

@@ -21,7 +21,7 @@ use std::fs::File;
 use std::path::PathBuf;
 use serde_derive::{Serialize, Deserialize};
 use primitives::storage::{StorageKey, StorageData};
-use runtime_primitives::{BuildStorage, StorageMap, ChildrenStorageMap};
+use runtime_primitives::{BuildStorage, StorageOverlay, ChildrenStorageOverlay};
 use serde_json as json;
 use crate::components::RuntimeGenesis;
 use network::Multiaddr;
@@ -65,7 +65,7 @@ impl<G: RuntimeGenesis> GenesisSource<G> {
 }
 
 impl<'a, G: RuntimeGenesis> BuildStorage for &'a ChainSpec<G> {
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String> {
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		match self.genesis.resolve()? {
 			Genesis::Runtime(gc) => gc.build_storage(),
 			Genesis::Raw(map) => Ok((map.into_iter().map(|(k, v)| (k.0, v.0)).collect(), Default::default())),

substrate/core/sr-primitives/src/lib.rs

@@ -89,11 +89,11 @@ use serde_derive::{Serialize, Deserialize};
 
 /// A set of key value pairs for storage.
 #[cfg(feature = "std")]
-pub type StorageMap = HashMap<Vec<u8>, Vec<u8>>;
+pub type StorageOverlay = HashMap<Vec<u8>, Vec<u8>>;
 
 /// A set of key value pairs for children storage;
 #[cfg(feature = "std")]
-pub type ChildrenStorageMap = HashMap<Vec<u8>, StorageMap>;
+pub type ChildrenStorageOverlay = HashMap<Vec<u8>, StorageOverlay>;
 
 /// Complex storage builder stuff.
 #[cfg(feature = "std")]
@@ -107,12 +107,12 @@ pub trait BuildStorage {
 		r
 	}
 	/// Build the storage out of this builder.
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String>;
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String>;
 }
 
 #[cfg(feature = "std")]
-impl BuildStorage for StorageMap {
-	fn build_storage(self) -> Result<(StorageMap, ChildrenStorageMap), String> {
+impl BuildStorage for StorageOverlay {
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		Ok((self, Default::default()))
 	}
 }
@@ -393,9 +393,9 @@ macro_rules! impl_outer_config {
 		}
 		#[cfg(any(feature = "std", test))]
 		impl $crate::BuildStorage for $main {
-			fn build_storage(self) -> ::std::result::Result<($crate::StorageMap, $crate::ChildrenStorageMap), String> {
-				let mut top = $crate::StorageMap::new();
-				let mut children = $crate::ChildrenStorageMap::new();
+			fn build_storage(self) -> ::std::result::Result<($crate::StorageOverlay, $crate::ChildrenStorageOverlay), String> {
+				let mut top = $crate::StorageOverlay::new();
+				let mut children = $crate::ChildrenStorageOverlay::new();
 				$(
 					if let Some(extra) = self.$snake {
 						let (other_top, other_children) = extra.build_storage()?;

substrate/core/test-client/src/lib.rs

@@ -35,7 +35,7 @@ pub use consensus;
 
 use std::sync::Arc;
 use primitives::Blake2Hasher;
-use runtime_primitives::StorageMap;
+use runtime_primitives::StorageOverlay;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT};
 use runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
 use keyring::Keyring;
@@ -126,7 +126,7 @@ fn genesis_config(support_changes_trie: bool) -> GenesisConfig {
 	], 1000)
 }
 
-fn genesis_storage(support_changes_trie: bool) -> StorageMap {
+fn genesis_storage(support_changes_trie: bool) -> StorageOverlay {
 	let mut storage = genesis_config(support_changes_trie).genesis_map();
 	let state_root = <<<runtime::Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(storage.clone().into_iter());
 	let block: runtime::Block = client::genesis::construct_genesis_block(state_root);