Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.

substrate/frame/contracts/src/account_db.rs

@@ -128,7 +128,7 @@ impl<T: Trait> AccountDb<T> for DirectAccountDb {
 		trie_id: Option<&TrieId>,
 		location: &StorageKey
 	) -> Option<Vec<u8>> {
-		trie_id.and_then(|id| child::get_raw(id, &blake2_256(location)))
+		trie_id.and_then(|id| child::get_raw(id, crate::trie_unique_id(&id[..]), &blake2_256(location)))
 	}
 	fn get_code_hash(&self, account: &T::AccountId) -> Option<CodeHash<T>> {
 		<ContractInfoOf<T>>::get(account).and_then(|i| i.as_alive().map(|i| i.code_hash))
@@ -173,13 +173,13 @@ impl<T: Trait> AccountDb<T> for DirectAccountDb {
 					(false, Some(info), _) => info,
 					// Existing contract is being removed.
 					(true, Some(info), None) => {
-						child::kill_storage(&info.trie_id);
+						child::kill_storage(&info.trie_id, info.child_trie_unique_id());
 						<ContractInfoOf<T>>::remove(&address);
 						continue;
 					}
 					// Existing contract is being replaced by a new one.
 					(true, Some(info), Some(code_hash)) => {
-						child::kill_storage(&info.trie_id);
+						child::kill_storage(&info.trie_id, info.child_trie_unique_id());
 						AliveContractInfo::<T> {
 							code_hash,
 							storage_size: T::StorageSizeOffset::get(),
@@ -217,14 +217,18 @@ impl<T: Trait> AccountDb<T> for DirectAccountDb {
 				}
 
 				for (k, v) in changed.storage.into_iter() {
-					if let Some(value) = child::get_raw(&new_info.trie_id[..], &blake2_256(&k)) {
+					if let Some(value) = child::get_raw(
+						&new_info.trie_id[..],
+						new_info.child_trie_unique_id(),
+						&blake2_256(&k),
+					) {
 						new_info.storage_size -= value.len() as u32;
 					}
 					if let Some(value) = v {
 						new_info.storage_size += value.len() as u32;
-						child::put_raw(&new_info.trie_id[..], &blake2_256(&k), &value[..]);
+						child::put_raw(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k), &value[..]);
 					} else {
-						child::kill(&new_info.trie_id[..], &blake2_256(&k));
+						child::kill(&new_info.trie_id[..], new_info.child_trie_unique_id(), &blake2_256(&k));
 					}
 				}
 

substrate/frame/contracts/src/lib.rs

@@ -223,6 +223,19 @@ pub struct RawAliveContractInfo<CodeHash, Balance, BlockNumber> {
 	pub last_write: Option<BlockNumber>,
 }
 
+impl<CodeHash, Balance, BlockNumber> RawAliveContractInfo<CodeHash, Balance, BlockNumber> {
+	/// Associated child trie unique id is built from the hash part of the trie id.
+	pub fn child_trie_unique_id(&self) -> child::ChildInfo {
+		trie_unique_id(&self.trie_id[..])
+	}
+}
+
+/// Associated child trie unique id is built from the hash part of the trie id.
+pub(crate) fn trie_unique_id(trie_id: &[u8]) -> child::ChildInfo {
+	let start = CHILD_STORAGE_KEY_PREFIX.len() + b"default:".len();
+	child::ChildInfo::new_default(&trie_id[start ..])
+}
+
 pub type TombstoneContractInfo<T> =
 	RawTombstoneContractInfo<<T as system::Trait>::Hash, <T as system::Trait>::Hashing>;
 
@@ -793,8 +806,17 @@ impl<T: Trait> Module<T> {
 
 		let key_values_taken = delta.iter()
 			.filter_map(|key| {
-				child::get_raw(&origin_contract.trie_id, &blake2_256(key)).map(|value| {
-					child::kill(&origin_contract.trie_id, &blake2_256(key));
+				child::get_raw(
+					&origin_contract.trie_id,
+					origin_contract.child_trie_unique_id(),
+					&blake2_256(key),
+				).map(|value| {
+					child::kill(
+						&origin_contract.trie_id,
+						origin_contract.child_trie_unique_id(),
+						&blake2_256(key),
+					);
+					
 					(key, value)
 				})
 			})
@@ -803,13 +825,20 @@ impl<T: Trait> Module<T> {
 		let tombstone = <TombstoneContractInfo<T>>::new(
 			// This operation is cheap enough because last_write (delta not included)
 			// is not this block as it has been checked earlier.
-			&sp_io::storage::child_root(&origin_contract.trie_id)[..],
+			&child::child_root(
+				&origin_contract.trie_id,
+			)[..],
 			code_hash,
 		);
 
 		if tombstone != dest_tombstone {
 			for (key, value) in key_values_taken {
-				child::put_raw(&origin_contract.trie_id, &blake2_256(key), &value);
+				child::put_raw(
+					&origin_contract.trie_id,
+					origin_contract.child_trie_unique_id(),
+					&blake2_256(key),
+					&value,
+				);
 			}
 
 			return Err("Tombstones don't match");
@@ -887,7 +916,7 @@ decl_storage! {
 impl<T: Trait> OnFreeBalanceZero<T::AccountId> for Module<T> {
 	fn on_free_balance_zero(who: &T::AccountId) {
 		if let Some(ContractInfo::Alive(info)) = <ContractInfoOf<T>>::take(who) {
-			child::kill_storage(&info.trie_id);
+			child::kill_storage(&info.trie_id, info.child_trie_unique_id());
 		}
 	}
 }

substrate/frame/contracts/src/rent.rs

@@ -19,6 +19,7 @@ use sp_runtime::traits::{Bounded, CheckedDiv, CheckedMul, Saturating, Zero,
 	SaturatedConversion};
 use support::traits::{Currency, ExistenceRequirement, Get, WithdrawReason, OnUnbalanced};
 use support::StorageMap;
+use support::storage::child;
 
 #[derive(PartialEq, Eq, Copy, Clone)]
 #[must_use]
@@ -99,7 +100,7 @@ fn try_evict_or_and_pay_rent<T: Trait>(
 	if balance < subsistence_threshold {
 		// The contract cannot afford to leave a tombstone, so remove the contract info altogether.
 		<ContractInfoOf<T>>::remove(account);
-		sp_io::storage::child_storage_kill(&contract.trie_id);
+		child::kill_storage(&contract.trie_id, contract.child_trie_unique_id());
 		return (RentOutcome::Evicted, None);
 	}
 
@@ -146,7 +147,9 @@ fn try_evict_or_and_pay_rent<T: Trait>(
 		// threshold, so it leaves a tombstone.
 
 		// Note: this operation is heavy.
-		let child_storage_root = sp_io::storage::child_root(&contract.trie_id);
+		let child_storage_root = child::child_root(
+			&contract.trie_id,
+		);
 
 		let tombstone = <TombstoneContractInfo<T>>::new(
 			&child_storage_root[..],
@@ -155,7 +158,7 @@ fn try_evict_or_and_pay_rent<T: Trait>(
 		let tombstone_info = ContractInfo::Tombstone(tombstone);
 		<ContractInfoOf<T>>::insert(account, &tombstone_info);
 
-		sp_io::storage::child_storage_kill(&contract.trie_id);
+		child::kill_storage(&contract.trie_id, contract.child_trie_unique_id());
 
 		return (RentOutcome::Evicted, Some(tombstone_info));
 	}

substrate/frame/support/procedural/src/storage/genesis_config/mod.rs

@@ -139,13 +139,10 @@ fn impl_build_storage(
 		#[cfg(feature = "std")]
 		impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause {
 			pub fn build_storage #fn_generic (&self) -> std::result::Result<
-				(
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				#scrate::sp_runtime::Storage,
 				String
 			> #fn_where_clause {
-				let mut storage = (Default::default(), Default::default());
+				let mut storage = Default::default();
 				self.assimilate_storage::<#fn_traitinstance>(&mut storage)?;
 				Ok(storage)
 			}
@@ -153,12 +150,9 @@ fn impl_build_storage(
 			/// Assimilate the storage for this module into pre-existing overlays.
 			pub fn assimilate_storage #fn_generic (
 				&self,
-				tuple_storage: &mut (
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				storage: &mut #scrate::sp_runtime::Storage,
 			) -> std::result::Result<(), String> #fn_where_clause {
-				#scrate::BasicExternalities::execute_with_storage(tuple_storage, || {
+				#scrate::BasicExternalities::execute_with_storage(storage, || {
 					#( #builder_blocks )*
 					Ok(())
 				})
@@ -171,10 +165,7 @@ fn impl_build_storage(
 		{
 			fn build_module_genesis_storage(
 				&self,
-				storage: &mut (
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				storage: &mut #scrate::sp_runtime::Storage,
 			) -> std::result::Result<(), String> {
 				self.assimilate_storage::<#fn_traitinstance> (storage)
 			}

substrate/frame/support/src/storage/child.rs

@@ -19,14 +19,29 @@
 //! This module is a currently only a variant of unhashed with additional `storage_key`.
 //! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to
 //! avoid collision from a resistant hash function (which unique implies)).
+//!
+//! A **key collision free** unique id is required as parameter to avoid key collision
+//! between child tries.
+//! This unique id management and generation responsability is delegated to pallet module.
 // NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>).
 
 use crate::sp_std::prelude::*;
 use codec::{Codec, Encode, Decode};
+pub use primitives::storage::ChildInfo;
 
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
-pub fn get<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
-	sp_io::storage::child_get(storage_key, key).and_then(|v| {
+pub fn get<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<T> {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_get(
+		storage_key,
+		data,
+		child_type,
+		key,
+	).and_then(|v| {
 		Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| {
 			// TODO #3700: error should be handleable.
 			runtime_print!("ERROR: Corrupted state in child trie at {:?}/{:?}", storage_key, key);
@@ -37,83 +52,178 @@ pub fn get<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
 /// explicit entry.
-pub fn get_or_default<T: Decode + Sized + Default>(storage_key: &[u8], key: &[u8]) -> T {
-	get(storage_key, key).unwrap_or_else(Default::default)
+pub fn get_or_default<T: Decode + Sized + Default>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> T {
+	get(storage_key, child_info, key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry.
-pub fn get_or<T: Decode + Sized>(storage_key: &[u8], key: &[u8], default_value: T) -> T {
-	get(storage_key, key).unwrap_or(default_value)
+pub fn get_or<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	default_value: T,
+) -> T {
+	get(storage_key, child_info, key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry.
 pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(
 	storage_key: &[u8],
+	child_info: ChildInfo,
 	key: &[u8],
 	default_value: F,
 ) -> T {
-	get(storage_key, key).unwrap_or_else(default_value)
+	get(storage_key, child_info, key).unwrap_or_else(default_value)
 }
 
 /// Put `value` in storage under `key`.
-pub fn put<T: Encode>(storage_key: &[u8], key: &[u8], value: &T) {
-	value.using_encoded(|slice| sp_io::storage::child_set(storage_key, key, slice));
+pub fn put<T: Encode>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	value: &T,
+) {
+	let (data, child_type) = child_info.info();
+	value.using_encoded(|slice|
+		sp_io::storage::child_set(
+			storage_key,
+			data,
+			child_type,
+			key,
+			slice,
+		)
+	);
 }
 
 /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
-pub fn take<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
-	let r = get(storage_key, key);
+pub fn take<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<T> {
+	let r = get(storage_key, child_info, key);
 	if r.is_some() {
-		kill(storage_key, key);
+		kill(storage_key, child_info, key);
 	}
 	r
 }
 
 /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
 /// the default for its type.
-pub fn take_or_default<T: Codec + Sized + Default>(storage_key: &[u8], key: &[u8]) -> T {
-	take(storage_key, key).unwrap_or_else(Default::default)
+pub fn take_or_default<T: Codec + Sized + Default>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> T {
+	take(storage_key, child_info, key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or<T: Codec + Sized>(storage_key: &[u8],key: &[u8], default_value: T) -> T {
-	take(storage_key, key).unwrap_or(default_value)
+pub fn take_or<T: Codec + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	default_value: T,
+) -> T {
+	take(storage_key, child_info, key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
 pub fn take_or_else<T: Codec + Sized, F: FnOnce() -> T>(
 	storage_key: &[u8],
+	child_info: ChildInfo,
 	key: &[u8],
 	default_value: F,
 ) -> T {
-	take(storage_key, key).unwrap_or_else(default_value)
+	take(storage_key, child_info, key).unwrap_or_else(default_value)
 }
 
 /// Check to see if `key` has an explicit entry in storage.
-pub fn exists(storage_key: &[u8], key: &[u8]) -> bool {
-	sp_io::storage::child_read(storage_key, key, &mut [0;0][..], 0).is_some()
+pub fn exists(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> bool {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_read(
+		storage_key, data, child_type,
+		key, &mut [0;0][..], 0,
+	).is_some()
 }
 
 /// Remove all `storage_key` key/values
-pub fn kill_storage(storage_key: &[u8]) {
-	sp_io::storage::child_storage_kill(storage_key)
+pub fn kill_storage(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_storage_kill(
+		storage_key,
+		data,
+		child_type,
+	)
 }
 
 /// Ensure `key` has no explicit entry in storage.
-pub fn kill(storage_key: &[u8], key: &[u8]) {
-	sp_io::storage::child_clear(storage_key, key);
+pub fn kill(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_clear(
+		storage_key,
+		data,
+		child_type,
+		key,
+	);
 }
 
 /// Get a Vec of bytes from storage.
-pub fn get_raw(storage_key: &[u8], key: &[u8]) -> Option<Vec<u8>> {
-	sp_io::storage::child_get(storage_key, key)
+pub fn get_raw(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<Vec<u8>> {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_get(
+		storage_key,
+		data,
+		child_type,
+		key,
+	)
 }
 
 /// Put a raw byte slice into storage.
-pub fn put_raw(storage_key: &[u8], key: &[u8], value: &[u8]) {
-	sp_io::storage::child_set(storage_key, key, value)
+pub fn put_raw(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	value: &[u8],
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_set(
+		storage_key,
+		data,
+		child_type,
+		key,
+		value,
+	)
+}
+
+/// Calculate current child root value.
+pub fn child_root(
+	storage_key: &[u8],
+) -> Vec<u8> {
+	sp_io::storage::child_root(
+		storage_key,
+	)
 }

substrate/frame/support/test/tests/instance.rs

@@ -300,7 +300,10 @@ fn new_test_ext() -> sp_io::TestExternalities {
 
 #[test]
 fn storage_instance_independance() {
-	let mut storage = Default::default();
+	let mut storage = primitives::storage::Storage {
+		top: std::collections::BTreeMap::new(),
+		children: std::collections::HashMap::new()
+	};
 	state_machine::BasicExternalities::execute_with_storage(&mut storage, || {
 		module2::Value::<Runtime>::put(0);
 		module2::Value::<Runtime, module2::Instance1>::put(0);
@@ -320,7 +323,7 @@ fn storage_instance_independance() {
 		module2::DoubleMap::<module2::Instance3>::insert(&0, &0, &0);
 	});
 	// 16 storage values + 4 linked_map head.
-	assert_eq!(storage.0.len(), 16 + 4);
+	assert_eq!(storage.top.len(), 16 + 4);
 }
 
 #[test]

substrate/frame/system/src/lib.rs

@@ -703,11 +703,14 @@ impl<T: Trait> Module<T> {
 	/// Get the basic externalities for this module, useful for tests.
 	#[cfg(any(feature = "std", test))]
 	pub fn externalities() -> TestExternalities {
-		TestExternalities::new((map![
-			<BlockHash<T>>::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(),
-			<Number<T>>::hashed_key().to_vec() => T::BlockNumber::one().encode(),
-			<ParentHash<T>>::hashed_key().to_vec() => [69u8; 32].encode()
-		], map![]))
+		TestExternalities::new(primitives::storage::Storage {
+			top: map![
+				<BlockHash<T>>::hashed_key_for(T::BlockNumber::zero()) => [69u8; 32].encode(),
+				<Number<T>>::hashed_key().to_vec() => T::BlockNumber::one().encode(),
+				<ParentHash<T>>::hashed_key().to_vec() => [69u8; 32].encode()
+			],
+			children: map![],
+		})
 	}
 
 	/// Set the block number to something in particular. Can be used as an alternative to