Child storage tests and genesis fix. (#3185)

* Using child storage, (srml-support only), test failing .

* fix simple tests.

* Enumerable by requiring owned struct (previous form only allow
&'static).
Broken tests are from genesis init.

* implement for_child_keys_with_prefix

* indent

* clear_child_prefix fix.

* clear_child_prefix fix 2.

* fix for storage_impl, if/when allowing child and not child this could be
reverted.

* Fix lot of urlinked child genesis, still need to look upon actual
genesis srml module code.
Probably still a lot of broken code needing debugging.

* switch well_known_key to their associated module child trie.
Fix a genesis init (balance).
Complete some testing.
Comment some tests before using.

* fixing test runtime child keys

* latest commit fix broken genesis init

* fix system balances child name.

* Important fix: storage_root from test externalities need children (it is
already the case for ext).

* executive root with child calculation

* Avoid empty trie on test ext.

* Symetric removal of key for system.

* commenting changes related tests.

* Remove child module specifics.

* fix issues.

* fix some formatting

* fix bench and bump runtime

* Remove extend_storage_overlays, assimilate_storage do the same as is
proper considering srml macro.

* Fix warning for assimilate.

* Removing kill as they do not impact any test cases.

* Use tuple of storage map instead of two parameters. This changes the
behavior of decl_storage genesis build closure (breaking api).

* Do not use build storage before assimilate.

* fix error

* Update core/state-machine/src/backend.rs

substrate/core/state-machine/src/testing.rs

@@ -17,7 +17,6 @@
 //! Test implementation for Externalities.
 
 use std::collections::{HashMap};
-use std::iter::FromIterator;
 use hash_db::Hasher;
 use crate::backend::{InMemory, Backend};
 use primitives::storage::well_known_keys::is_child_storage_key;
@@ -46,22 +45,12 @@ pub struct TestExternalities<H: Hasher, N: ChangesTrieBlockNumber> {
 
 impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 	/// Create a new instance of `TestExternalities` with storage.
-	pub fn new(storage: HashMap<Vec<u8>, Vec<u8>>) -> Self {
-		Self::new_with_children((storage, Default::default()))
-	}
-
-	/// Create a new instance of `TestExternalities` with storage and children.
-	pub fn new_with_children(storage: StorageTuple) -> Self {
-		Self::new_with_code_with_children(&[], storage)
+	pub fn new(storage: StorageTuple) -> Self {
+		Self::new_with_code(&[], storage)
 	}
 
 	/// Create a new instance of `TestExternalities` with code and storage.
-	pub fn new_with_code(code: &[u8], storage: HashMap<Vec<u8>, Vec<u8>>) -> Self {
-		Self::new_with_code_with_children(code, (storage, Default::default()))
-	}
-
-	/// Create a new instance of `TestExternalities` with code, storage and children.
-	pub fn new_with_code_with_children(code: &[u8], mut storage: StorageTuple) -> Self {
+	pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self {
 		let mut overlay = OverlayedChanges::default();
 
 		assert!(storage.0.keys().all(|key| !is_child_storage_key(key)));
@@ -137,25 +126,13 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> PartialEq for TestExternalities<H, N>
 	}
 }
 
-impl<H: Hasher, N: ChangesTrieBlockNumber> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H, N> {
-	fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
-		Self::new(iter.into_iter().collect())
-	}
-}
-
 impl<H: Hasher, N: ChangesTrieBlockNumber> Default for TestExternalities<H, N> {
 	fn default() -> Self { Self::new(Default::default()) }
 }
 
-impl<H: Hasher, N: ChangesTrieBlockNumber> From<HashMap<Vec<u8>, Vec<u8>>> for TestExternalities<H, N> {
-	fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
-		Self::from_iter(hashmap)
-	}
-}
-
 impl<H: Hasher, N: ChangesTrieBlockNumber> From<StorageTuple> for TestExternalities<H, N> {
 	fn from(storage: StorageTuple) -> Self {
-		Self::new_with_children(storage)
+		Self::new(storage)
 	}
 }
 
@@ -184,6 +161,13 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
 			)
 	}
 
+	fn original_child_storage(&self, storage_key: ChildStorageKey<H>, key: &[u8]) -> Option<Vec<u8>> {
+		self.backend
+			.child_storage(storage_key.as_ref(), key)
+			.map(|x| x.map(|x| x.to_vec()))
+			.expect(EXT_NOT_ALLOWED_TO_FAIL)
+	}
+
 	fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
 		if is_child_storage_key(&key) {
 			panic!("Refuse to directly set child storage key");
@@ -225,20 +209,45 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
 		});
 	}
 
+	fn clear_child_prefix(&mut self, storage_key: ChildStorageKey<H>, prefix: &[u8]) {
+
+		self.overlay.clear_child_prefix(storage_key.as_ref(), prefix);
+
+		let backend = &self.backend;
+		let overlay = &mut self.overlay;
+		backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| {
+			overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None);
+		});
+	}
+
 	fn chain_id(&self) -> u64 { 42 }
 
 	fn storage_root(&mut self) -> H::Out {
+
+		let child_storage_keys =
+			self.overlay.prospective.children.keys()
+				.chain(self.overlay.committed.children.keys());
+
+		let child_delta_iter = child_storage_keys.map(|storage_key|
+			(storage_key.clone(), self.overlay.committed.children.get(storage_key)
+				.into_iter()
+				.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone())))
+				.chain(self.overlay.prospective.children.get(storage_key)
+					.into_iter()
+					.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))))));
+
+
 		// compute and memoize
 		let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))
 			.chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone())));
+		self.backend.full_storage_root(delta, child_delta_iter).0
 
-		self.backend.storage_root(delta).0
 	}
 
 	fn child_storage_root(&mut self, storage_key: ChildStorageKey<H>) -> Vec<u8> {
 		let storage_key = storage_key.as_ref();
 
-		let (root, _, _) = {
+		let (root, is_empty, _) = {
 			let delta = self.overlay.committed.children.get(storage_key)
 				.into_iter()
 				.flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone())))
@@ -248,7 +257,11 @@ impl<H, N> Externalities<H> for TestExternalities<H, N>
 
 			self.backend.child_storage_root(storage_key, delta)
 		};
-		self.overlay.set_storage(storage_key.into(), Some(root.clone()));
+		if is_empty {
+			self.overlay.set_storage(storage_key.into(), None);
+		} else {
+			self.overlay.set_storage(storage_key.into(), Some(root.clone()));
+		}
 		root
 	}