Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.

substrate/frame/support/procedural/src/storage/genesis_config/mod.rs

@@ -139,13 +139,10 @@ fn impl_build_storage(
 		#[cfg(feature = "std")]
 		impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause {
 			pub fn build_storage #fn_generic (&self) -> std::result::Result<
-				(
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				#scrate::sp_runtime::Storage,
 				String
 			> #fn_where_clause {
-				let mut storage = (Default::default(), Default::default());
+				let mut storage = Default::default();
 				self.assimilate_storage::<#fn_traitinstance>(&mut storage)?;
 				Ok(storage)
 			}
@@ -153,12 +150,9 @@ fn impl_build_storage(
 			/// Assimilate the storage for this module into pre-existing overlays.
 			pub fn assimilate_storage #fn_generic (
 				&self,
-				tuple_storage: &mut (
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				storage: &mut #scrate::sp_runtime::Storage,
 			) -> std::result::Result<(), String> #fn_where_clause {
-				#scrate::BasicExternalities::execute_with_storage(tuple_storage, || {
+				#scrate::BasicExternalities::execute_with_storage(storage, || {
 					#( #builder_blocks )*
 					Ok(())
 				})
@@ -171,10 +165,7 @@ fn impl_build_storage(
 		{
 			fn build_module_genesis_storage(
 				&self,
-				storage: &mut (
-					#scrate::sp_runtime::StorageOverlay,
-					#scrate::sp_runtime::ChildrenStorageOverlay,
-				),
+				storage: &mut #scrate::sp_runtime::Storage,
 			) -> std::result::Result<(), String> {
 				self.assimilate_storage::<#fn_traitinstance> (storage)
 			}

substrate/frame/support/src/storage/child.rs

@@ -19,14 +19,29 @@
 //! This module is a currently only a variant of unhashed with additional `storage_key`.
 //! Note that `storage_key` must be unique and strong (strong in the sense of being long enough to
 //! avoid collision from a resistant hash function (which unique implies)).
+//!
+//! A **key collision free** unique id is required as parameter to avoid key collision
+//! between child tries.
+//! This unique id management and generation responsability is delegated to pallet module.
 // NOTE: could replace unhashed by having only one kind of storage (root being null storage key (storage_key can become Option<&[u8]>).
 
 use crate::sp_std::prelude::*;
 use codec::{Codec, Encode, Decode};
+pub use primitives::storage::ChildInfo;
 
 /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry.
-pub fn get<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
-	sp_io::storage::child_get(storage_key, key).and_then(|v| {
+pub fn get<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<T> {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_get(
+		storage_key,
+		data,
+		child_type,
+		key,
+	).and_then(|v| {
 		Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| {
 			// TODO #3700: error should be handleable.
 			runtime_print!("ERROR: Corrupted state in child trie at {:?}/{:?}", storage_key, key);
@@ -37,83 +52,178 @@ pub fn get<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
 
 /// Return the value of the item in storage under `key`, or the type's default if there is no
 /// explicit entry.
-pub fn get_or_default<T: Decode + Sized + Default>(storage_key: &[u8], key: &[u8]) -> T {
-	get(storage_key, key).unwrap_or_else(Default::default)
+pub fn get_or_default<T: Decode + Sized + Default>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> T {
+	get(storage_key, child_info, key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry.
-pub fn get_or<T: Decode + Sized>(storage_key: &[u8], key: &[u8], default_value: T) -> T {
-	get(storage_key, key).unwrap_or(default_value)
+pub fn get_or<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	default_value: T,
+) -> T {
+	get(storage_key, child_info, key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry.
 pub fn get_or_else<T: Decode + Sized, F: FnOnce() -> T>(
 	storage_key: &[u8],
+	child_info: ChildInfo,
 	key: &[u8],
 	default_value: F,
 ) -> T {
-	get(storage_key, key).unwrap_or_else(default_value)
+	get(storage_key, child_info, key).unwrap_or_else(default_value)
 }
 
 /// Put `value` in storage under `key`.
-pub fn put<T: Encode>(storage_key: &[u8], key: &[u8], value: &T) {
-	value.using_encoded(|slice| sp_io::storage::child_set(storage_key, key, slice));
+pub fn put<T: Encode>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	value: &T,
+) {
+	let (data, child_type) = child_info.info();
+	value.using_encoded(|slice|
+		sp_io::storage::child_set(
+			storage_key,
+			data,
+			child_type,
+			key,
+			slice,
+		)
+	);
 }
 
 /// Remove `key` from storage, returning its value if it had an explicit entry or `None` otherwise.
-pub fn take<T: Decode + Sized>(storage_key: &[u8], key: &[u8]) -> Option<T> {
-	let r = get(storage_key, key);
+pub fn take<T: Decode + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<T> {
+	let r = get(storage_key, child_info, key);
 	if r.is_some() {
-		kill(storage_key, key);
+		kill(storage_key, child_info, key);
 	}
 	r
 }
 
 /// Remove `key` from storage, returning its value, or, if there was no explicit entry in storage,
 /// the default for its type.
-pub fn take_or_default<T: Codec + Sized + Default>(storage_key: &[u8], key: &[u8]) -> T {
-	take(storage_key, key).unwrap_or_else(Default::default)
+pub fn take_or_default<T: Codec + Sized + Default>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> T {
+	take(storage_key, child_info, key).unwrap_or_else(Default::default)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
-pub fn take_or<T: Codec + Sized>(storage_key: &[u8],key: &[u8], default_value: T) -> T {
-	take(storage_key, key).unwrap_or(default_value)
+pub fn take_or<T: Codec + Sized>(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	default_value: T,
+) -> T {
+	take(storage_key, child_info, key).unwrap_or(default_value)
 }
 
 /// Return the value of the item in storage under `key`, or `default_value()` if there is no
 /// explicit entry. Ensure there is no explicit entry on return.
 pub fn take_or_else<T: Codec + Sized, F: FnOnce() -> T>(
 	storage_key: &[u8],
+	child_info: ChildInfo,
 	key: &[u8],
 	default_value: F,
 ) -> T {
-	take(storage_key, key).unwrap_or_else(default_value)
+	take(storage_key, child_info, key).unwrap_or_else(default_value)
 }
 
 /// Check to see if `key` has an explicit entry in storage.
-pub fn exists(storage_key: &[u8], key: &[u8]) -> bool {
-	sp_io::storage::child_read(storage_key, key, &mut [0;0][..], 0).is_some()
+pub fn exists(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> bool {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_read(
+		storage_key, data, child_type,
+		key, &mut [0;0][..], 0,
+	).is_some()
 }
 
 /// Remove all `storage_key` key/values
-pub fn kill_storage(storage_key: &[u8]) {
-	sp_io::storage::child_storage_kill(storage_key)
+pub fn kill_storage(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_storage_kill(
+		storage_key,
+		data,
+		child_type,
+	)
 }
 
 /// Ensure `key` has no explicit entry in storage.
-pub fn kill(storage_key: &[u8], key: &[u8]) {
-	sp_io::storage::child_clear(storage_key, key);
+pub fn kill(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_clear(
+		storage_key,
+		data,
+		child_type,
+		key,
+	);
 }
 
 /// Get a Vec of bytes from storage.
-pub fn get_raw(storage_key: &[u8], key: &[u8]) -> Option<Vec<u8>> {
-	sp_io::storage::child_get(storage_key, key)
+pub fn get_raw(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+) -> Option<Vec<u8>> {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_get(
+		storage_key,
+		data,
+		child_type,
+		key,
+	)
 }
 
 /// Put a raw byte slice into storage.
-pub fn put_raw(storage_key: &[u8], key: &[u8], value: &[u8]) {
-	sp_io::storage::child_set(storage_key, key, value)
+pub fn put_raw(
+	storage_key: &[u8],
+	child_info: ChildInfo,
+	key: &[u8],
+	value: &[u8],
+) {
+	let (data, child_type) = child_info.info();
+	sp_io::storage::child_set(
+		storage_key,
+		data,
+		child_type,
+		key,
+		value,
+	)
+}
+
+/// Calculate current child root value.
+pub fn child_root(
+	storage_key: &[u8],
+) -> Vec<u8> {
+	sp_io::storage::child_root(
+		storage_key,
+	)
 }

substrate/frame/support/test/tests/instance.rs

@@ -300,7 +300,10 @@ fn new_test_ext() -> sp_io::TestExternalities {
 
 #[test]
 fn storage_instance_independance() {
-	let mut storage = Default::default();
+	let mut storage = primitives::storage::Storage {
+		top: std::collections::BTreeMap::new(),
+		children: std::collections::HashMap::new()
+	};
 	state_machine::BasicExternalities::execute_with_storage(&mut storage, || {
 		module2::Value::<Runtime>::put(0);
 		module2::Value::<Runtime, module2::Instance1>::put(0);
@@ -320,7 +323,7 @@ fn storage_instance_independance() {
 		module2::DoubleMap::<module2::Instance3>::insert(&0, &0, &0);
 	});
 	// 16 storage values + 4 linked_map head.
-	assert_eq!(storage.0.len(), 16 + 4);
+	assert_eq!(storage.top.len(), 16 + 4);
 }
 
 #[test]