Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.

substrate/primitives/state-machine/src/testing.rs

@@ -16,7 +16,7 @@
 
 //! Test implementation for Externalities.
 
-use std::{collections::{HashMap, BTreeMap}, any::{Any, TypeId}};
+use std::any::{Any, TypeId};
 use hash_db::Hasher;
 use crate::{
 	backend::{InMemory, Backend}, OverlayedChanges,
@@ -28,15 +28,14 @@ use crate::{
 };
 use primitives::{
 	storage::{
-		well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}
+		well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
+		Storage,
 	},
 	hash::H256, Blake2Hasher,
 };
 use codec::Encode;
 use externalities::{Extensions, Extension};
 
-type StorageTuple = (BTreeMap<Vec<u8>, Vec<u8>>, HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>);
-
 /// Simple HashMap-based Externalities impl.
 pub struct TestExternalities<H: Hasher<Out=H256>=Blake2Hasher, N: ChangesTrieBlockNumber=u64> {
 	overlay: OverlayedChanges,
@@ -57,42 +56,37 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 	}
 
 	/// Create a new instance of `TestExternalities` with storage.
-	pub fn new(storage: StorageTuple) -> Self {
+	pub fn new(storage: Storage) -> Self {
 		Self::new_with_code(&[], storage)
 	}
 
 	/// Create a new instance of `TestExternalities` with code and storage.
-	pub fn new_with_code(code: &[u8], mut storage: StorageTuple) -> Self {
+	pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self {
 		let mut overlay = OverlayedChanges::default();
 
-		assert!(storage.0.keys().all(|key| !is_child_storage_key(key)));
-		assert!(storage.1.keys().all(|key| is_child_storage_key(key)));
+		assert!(storage.top.keys().all(|key| !is_child_storage_key(key)));
+		assert!(storage.children.keys().all(|key| is_child_storage_key(key)));
 
 		super::set_changes_trie_config(
 			&mut overlay,
-			storage.0.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(),
+			storage.top.get(&CHANGES_TRIE_CONFIG.to_vec()).cloned(),
 			false,
 		).expect("changes trie configuration is correct in test env; qed");
 
-		storage.0.insert(HEAP_PAGES.to_vec(), 8u64.encode());
-		storage.0.insert(CODE.to_vec(), code.to_vec());
-
-		let backend: HashMap<_, _> = storage.1.into_iter()
-			.map(|(keyspace, map)| (Some(keyspace), map))
-			.chain(Some((None, storage.0)).into_iter())
-			.collect();
+		storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode());
+		storage.top.insert(CODE.to_vec(), code.to_vec());
 
 		TestExternalities {
 			overlay,
 			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
-			backend: backend.into(),
+			backend: storage.into(),
 			extensions: Default::default(),
 		}
 	}
 
 	/// Insert key/value into backend
 	pub fn insert(&mut self, k: Vec<u8>, v: Vec<u8>) {
-		self.backend = self.backend.update(vec![(None, k, Some(v))]);
+		self.backend = self.backend.update(vec![(None, vec![(k, Some(v))])]);
 	}
 
 	/// Registers the given extension for this instance.
@@ -107,19 +101,23 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> TestExternalities<H, N> {
 
 	/// Return a new backend with all pending value.
 	pub fn commit_all(&self) -> InMemory<H> {
-		let top = self.overlay.committed.top.clone().into_iter()
+		let top: Vec<_> = self.overlay.committed.top.clone().into_iter()
 			.chain(self.overlay.prospective.top.clone().into_iter())
-			.map(|(k, v)| (None, k, v.value));
+			.map(|(k, v)| (k, v.value)).collect();
+		let mut transaction = vec![(None, top)];
 
-		let children = self.overlay.committed.children.clone().into_iter()
+		self.overlay.committed.children.clone().into_iter()
 			.chain(self.overlay.prospective.children.clone().into_iter())
-			.flat_map(|(keyspace, map)| {
-				map.into_iter()
-					.map(|(k, v)| (Some(keyspace.clone()), k, v.value))
-					.collect::<Vec<_>>()
+			.for_each(|(keyspace, (map, child_info))| {
+				transaction.push((
+					Some((keyspace, child_info)),
+					map.into_iter()
+						.map(|(k, v)| (k, v.value))
+						.collect::<Vec<_>>(),
+				))
 			});
 
-		self.backend.update(top.chain(children).collect())
+		self.backend.update(transaction)
 	}
 
 	/// Execute the given closure while `self` is set as externalities.
@@ -149,8 +147,8 @@ impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> Default for TestExternaliti
 	fn default() -> Self { Self::new(Default::default()) }
 }
 
-impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> From<StorageTuple> for TestExternalities<H, N> {
-	fn from(storage: StorageTuple) -> Self {
+impl<H: Hasher<Out=H256>, N: ChangesTrieBlockNumber> From<Storage> for TestExternalities<H, N> {
+	fn from(storage: Storage) -> Self {
 		Self::new(storage)
 	}
 }