Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.

substrate/test-utils/runtime/src/genesismap.rs

@@ -16,11 +16,12 @@
 
 //! Tool for creating the genesis block.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::BTreeMap;
 use sp_io::hashing::{blake2_256, twox_128};
 use super::{AuthorityId, AccountId, WASM_BINARY, system};
 use codec::{Encode, KeyedVec, Joiner};
-use primitives::{ChangesTrieConfiguration, map, storage::well_known_keys};
+use primitives::{ChangesTrieConfiguration, map};
+use primitives::storage::{well_known_keys, Storage};
 use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT};
 
 /// Configuration of a general Substrate test genesis block.
@@ -30,8 +31,7 @@ pub struct GenesisConfig {
 	balances: Vec<(AccountId, u64)>,
 	heap_pages_override: Option<u64>,
 	/// Additional storage key pairs that will be added to the genesis map.
-	extra_storage: BTreeMap<Vec<u8>, Vec<u8>>,
-	child_extra_storage: HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
+	extra_storage: Storage,
 }
 
 impl GenesisConfig {
@@ -41,8 +41,7 @@ impl GenesisConfig {
 		endowed_accounts: Vec<AccountId>,
 		balance: u64,
 		heap_pages_override: Option<u64>,
-		extra_storage: BTreeMap<Vec<u8>, Vec<u8>>,
-		child_extra_storage: HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
+		extra_storage: Storage, 
 	) -> Self {
 		GenesisConfig {
 			changes_trie_config: match support_changes_trie {
@@ -53,14 +52,10 @@ impl GenesisConfig {
 			balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(),
 			heap_pages_override,
 			extra_storage,
-			child_extra_storage,
 		}
 	}
 
-	pub fn genesis_map(&self) -> (
-		BTreeMap<Vec<u8>, Vec<u8>>,
-		HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
-	) {
+	pub fn genesis_map(&self) -> Storage {
 		let wasm_runtime = WASM_BINARY.to_vec();
 		let mut map: BTreeMap<Vec<u8>, Vec<u8>> = self.balances.iter()
 			.map(|&(ref account, balance)| (account.to_keyed_vec(b"balance:"), vec![].and(&balance)))
@@ -78,10 +73,10 @@ impl GenesisConfig {
 		}
 		map.insert(twox_128(&b"sys:auth"[..])[..].to_vec(), self.authorities.encode());
 		// Add the extra storage entries.
-		map.extend(self.extra_storage.clone().into_iter());
+		map.extend(self.extra_storage.top.clone().into_iter());
 
 		// Assimilate the system genesis config.
-		let mut storage = (map, self.child_extra_storage.clone());
+		let mut storage = Storage { top: map, children: self.extra_storage.children.clone()};
 		let mut config = system::GenesisConfig::default();
 		config.authorities = self.authorities.clone();
 		config.assimilate_storage(&mut storage).expect("Adding `system::GensisConfig` to the genesis");
@@ -91,23 +86,22 @@ impl GenesisConfig {
 }
 
 pub fn insert_genesis_block(
-	storage: &mut (
-		BTreeMap<Vec<u8>, Vec<u8>>,
-		HashMap<Vec<u8>, BTreeMap<Vec<u8>, Vec<u8>>>,
-	)
+	storage: &mut Storage,
 ) -> primitives::hash::H256 {
-	let child_roots = storage.1.iter().map(|(sk, child_map)| {
+	let child_roots = storage.children.iter().map(|(sk, child_content)| {
 		let state_root = <<<crate::Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
-			child_map.clone().into_iter().collect(),
+			child_content.data.clone().into_iter().collect(),
 		);
 		(sk.clone(), state_root.encode())
 	});
+	// add child roots to storage
+	storage.top.extend(child_roots);
 	let state_root = <<<crate::Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
-		storage.0.clone().into_iter().chain(child_roots).collect()
+		storage.top.clone().into_iter().collect()
 	);
 	let block: crate::Block = sc_client::genesis::construct_genesis_block(state_root);
 	let genesis_hash = block.header.hash();
-	storage.0.extend(additional_storage_with_genesis(&block));
+	storage.top.extend(additional_storage_with_genesis(&block));
 	genesis_hash
 }
 

substrate/test-utils/runtime/src/lib.rs

@@ -50,6 +50,7 @@ use runtime_version::NativeVersion;
 use runtime_support::{impl_outer_origin, parameter_types, weights::Weight};
 use inherents::{CheckInherentsResult, InherentData};
 use cfg_if::cfg_if;
+use primitives::storage::ChildType;
 
 // Ensure Babe and Aura use the same crypto to simplify things a bit.
 pub use babe_primitives::AuthorityId;
@@ -910,21 +911,37 @@ fn test_read_storage() {
 
 fn test_read_child_storage() {
 	const CHILD_KEY: &[u8] = b":child_storage:default:read_child_storage";
+	const UNIQUE_ID: &[u8] = b":unique_id";
 	const KEY: &[u8] = b":read_child_storage";
-	sp_io::storage::child_set(CHILD_KEY, KEY, b"test");
+	sp_io::storage::child_set(
+		CHILD_KEY,
+		UNIQUE_ID,
+		ChildType::CryptoUniqueId as u32,
+		KEY,
+		b"test",
+	);
 
 	let mut v = [0u8; 4];
 	let r = sp_io::storage::child_read(
 		CHILD_KEY,
+		UNIQUE_ID,
+		ChildType::CryptoUniqueId as u32,
 		KEY,
 		&mut v,
-		0
+		0,
 	);
 	assert_eq!(r, Some(4));
 	assert_eq!(&v, b"test");
 
 	let mut v = [0u8; 4];
-	let r = sp_io::storage::child_read(CHILD_KEY, KEY, &mut v, 8);
+	let r = sp_io::storage::child_read(
+		CHILD_KEY,
+		UNIQUE_ID,
+		ChildType::CryptoUniqueId as u32,
+		KEY,
+		&mut v,
+		8,
+	);
 	assert_eq!(r, Some(4));
 	assert_eq!(&v, &[0, 0, 0, 0]);
 }

substrate/test-utils/runtime/src/system.rs

@@ -361,16 +361,16 @@ mod tests {
 		];
 		TestExternalities::new_with_code(
 			WASM_BINARY,
-			(
-				map![
+			primitives::storage::Storage {
+				top: map![
 					twox_128(b"latest").to_vec() => vec![69u8; 32],
 					twox_128(b"sys:auth").to_vec() => authorities.encode(),
 					blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => {
 						vec![111u8, 0, 0, 0, 0, 0, 0, 0]
 					}
 				],
-				map![],
-			)
+				children: map![],
+			},
 		)
 	}