Fix key collision for child trie (#4162)

* In progress, runtime io must switch to future proof root +
child_specific (unique id) + u32 type.

* Switch interface, sr-io seems ok, rpc could use similar interface to
sr-io, genesis json broken if there is child trie in existing encoding
genesis.

* test from previous implementation.

* fix proving test.

* Restore Keyspacedb from other branch, only apply to child trie.

* Removing unneeded child_info from child root (child info are stored
if things changed, otherwhise the root does not change).

* Switch rpc to use same format as ext: more future proof.

* use root from child info for trie backend essence.

* Breaking long lines.

* Update doc and clean pr a bit.

* fix error type

* Restore removed doc on merge and update sr-io doc.

* Switch child storage api to use directly unique id, if managed id
where to be put in place, the api will change at this time.

* Clean deprecated host interface from child.

* Removing assertion on child info (can fail depending on root
memoization).

* merging child info in the overlay when possible.

* child iteration by prefix using child_info.

* Using ChainInfo in frame support. ChainInfo gets redesign to avoid
buffers allocation on every calls.

* Add length of root to the data of child info.

* comments

* Encode compact.

* Remove child info with root.

* Fix try_update condition.

* Comment Ext child root caching.

* Replace tuples by struct with field

* remove StorageTuple alias.

* Fix doc tests, and remove StorageOverlay and ChildStorageOverlay
aliases.

substrate/client/rpc/src/state/mod.rs

@@ -95,6 +95,8 @@ pub trait StateBackend<B, E, Block: BlockT, RA>: Send + Sync + 'static
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		prefix: StorageKey,
 	) -> FutureResult<Vec<StorageKey>>;
 
@@ -103,6 +105,8 @@ pub trait StateBackend<B, E, Block: BlockT, RA>: Send + Sync + 'static
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<StorageData>>;
 
@@ -111,6 +115,8 @@ pub trait StateBackend<B, E, Block: BlockT, RA>: Send + Sync + 'static
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<Block::Hash>>;
 
@@ -119,9 +125,11 @@ pub trait StateBackend<B, E, Block: BlockT, RA>: Send + Sync + 'static
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<u64>> {
-		Box::new(self.child_storage(block, child_storage_key, key)
+		Box::new(self.child_storage(block, child_storage_key, child_info, child_type, key)
 			.map(|x| x.map(|x| x.0.len() as u64)))
 	}
 
@@ -256,37 +264,45 @@ impl<B, E, Block, RA> StateApi<Block::Hash> for State<B, E, Block, RA>
 	fn child_storage(
 		&self,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 		block: Option<Block::Hash>
 	) -> FutureResult<Option<StorageData>> {
-		self.backend.child_storage(block, child_storage_key, key)
+		self.backend.child_storage(block, child_storage_key, child_info, child_type, key)
 	}
 
 	fn child_storage_keys(
 		&self,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key_prefix: StorageKey,
 		block: Option<Block::Hash>
 	) -> FutureResult<Vec<StorageKey>> {
-		self.backend.child_storage_keys(block, child_storage_key, key_prefix)
+		self.backend.child_storage_keys(block, child_storage_key, child_info, child_type, key_prefix)
 	}
 
 	fn child_storage_hash(
 		&self,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 		block: Option<Block::Hash>
 	) -> FutureResult<Option<Block::Hash>> {
-		self.backend.child_storage_hash(block, child_storage_key, key)
+		self.backend.child_storage_hash(block, child_storage_key, child_info, child_type, key)
 	}
 
 	fn child_storage_size(
 		&self,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 		block: Option<Block::Hash>
 	) -> FutureResult<Option<u64>> {
-		self.backend.child_storage_size(block, child_storage_key, key)
+		self.backend.child_storage_size(block, child_storage_key, child_info, child_type, key)
 	}
 
 	fn metadata(&self, block: Option<Block::Hash>) -> FutureResult<Bytes> {
@@ -335,3 +351,9 @@ impl<B, E, Block, RA> StateApi<Block::Hash> for State<B, E, Block, RA>
 fn client_err(err: sp_blockchain::Error) -> Error {
 	Error::Client(Box::new(err))
 }
+
+const CHILD_RESOLUTION_ERROR: &str = "Unexpected child info and type";
+
+fn child_resolution_error() -> sp_blockchain::Error {
+	sp_blockchain::Error::Msg(CHILD_RESOLUTION_ERROR.to_string())
+}

substrate/client/rpc/src/state/state_full.rs

@@ -36,7 +36,8 @@ use client::{
 	Client, CallExecutor, BlockchainEvents, 
 };
 use primitives::{
-	H256, Blake2Hasher, Bytes, storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet},
+	H256, Blake2Hasher, Bytes,
+	storage::{well_known_keys, StorageKey, StorageData, StorageChangeSet, ChildInfo},
 };
 use runtime_version::RuntimeVersion;
 use state_machine::ExecutionStrategy;
@@ -47,7 +48,7 @@ use sp_runtime::{
 
 use sp_api::Metadata;
 
-use super::{StateBackend, error::{FutureResult, Error, Result}, client_err};
+use super::{StateBackend, error::{FutureResult, Error, Result}, client_err, child_resolution_error};
 
 /// Ranges to query in state_queryStorage.
 struct QueryStorageRange<Block: BlockT> {
@@ -287,11 +288,19 @@ impl<B, E, Block, RA> StateBackend<B, E, Block, RA> for FullState<B, E, Block, R
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		prefix: StorageKey,
 	) -> FutureResult<Vec<StorageKey>> {
 		Box::new(result(
 			self.block_or_best(block)
-				.and_then(|block| self.client.child_storage_keys(&BlockId::Hash(block), &child_storage_key, &prefix))
+				.and_then(|block| self.client.child_storage_keys(
+					&BlockId::Hash(block),
+					&child_storage_key,
+					ChildInfo::resolve_child_info(child_type, &child_info.0[..])
+						.ok_or_else(child_resolution_error)?,
+					&prefix,
+				))
 				.map_err(client_err)))
 	}
 
@@ -299,11 +308,19 @@ impl<B, E, Block, RA> StateBackend<B, E, Block, RA> for FullState<B, E, Block, R
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<StorageData>> {
 		Box::new(result(
 			self.block_or_best(block)
-				.and_then(|block| self.client.child_storage(&BlockId::Hash(block), &child_storage_key, &key))
+				.and_then(|block| self.client.child_storage(
+					&BlockId::Hash(block),
+					&child_storage_key,
+					ChildInfo::resolve_child_info(child_type, &child_info.0[..])
+						.ok_or_else(child_resolution_error)?,
+					&key,
+				))
 				.map_err(client_err)))
 	}
 
@@ -311,11 +328,19 @@ impl<B, E, Block, RA> StateBackend<B, E, Block, RA> for FullState<B, E, Block, R
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<Block::Hash>> {
 		Box::new(result(
 			self.block_or_best(block)
-				.and_then(|block| self.client.child_storage_hash(&BlockId::Hash(block), &child_storage_key, &key))
+				.and_then(|block| self.client.child_storage_hash(
+					&BlockId::Hash(block),
+					&child_storage_key,
+					ChildInfo::resolve_child_info(child_type, &child_info.0[..])
+						.ok_or_else(child_resolution_error)?,
+					&key,
+				))
 				.map_err(client_err)))
 	}
 

substrate/client/rpc/src/state/state_light.rs

@@ -236,6 +236,8 @@ impl<Block, F, B, E, RA> StateBackend<B, E, Block, RA> for LightState<Block, F,
 		&self,
 		_block: Option<Block::Hash>,
 		_child_storage_key: StorageKey,
+		_child_info: StorageKey,
+		_child_type: u32,
 		_prefix: StorageKey,
 	) -> FutureResult<Vec<StorageKey>> {
 		Box::new(result(Err(client_err(ClientError::NotAvailableOnLightClient))))
@@ -245,6 +247,8 @@ impl<Block, F, B, E, RA> StateBackend<B, E, Block, RA> for LightState<Block, F,
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<StorageData>> {
 		let block = self.block_or_best(block);
@@ -255,6 +259,8 @@ impl<Block, F, B, E, RA> StateBackend<B, E, Block, RA> for LightState<Block, F,
 					block,
 					header,
 					storage_key: child_storage_key.0,
+					child_info: child_info.0,
+					child_type,
 					keys: vec![key.0.clone()],
 					retry_count: Default::default(),
 				}).then(move |result| ready(result
@@ -275,10 +281,12 @@ impl<Block, F, B, E, RA> StateBackend<B, E, Block, RA> for LightState<Block, F,
 		&self,
 		block: Option<Block::Hash>,
 		child_storage_key: StorageKey,
+		child_info: StorageKey,
+		child_type: u32,
 		key: StorageKey,
 	) -> FutureResult<Option<Block::Hash>> {
 		Box::new(self
-			.child_storage(block, child_storage_key, key)
+			.child_storage(block, child_storage_key, child_info, child_type, key)
 			.and_then(|maybe_storage|
 				result(Ok(maybe_storage.map(|storage| Blake2Hasher::hash(&storage.0))))
 			)

substrate/client/rpc/src/state/tests.rs

@@ -21,7 +21,7 @@ use self::error::Error;
 use std::sync::Arc;
 use assert_matches::assert_matches;
 use futures01::stream::Stream;
-use primitives::storage::well_known_keys;
+use primitives::storage::{well_known_keys, ChildInfo};
 use primitives::hash::H256;
 use sp_io::hashing::blake2_256;
 use test_client::{
@@ -30,6 +30,8 @@ use test_client::{
 	runtime,
 };
 
+const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"unique_id");
+
 #[test]
 fn should_return_storage() {
 	const KEY: &[u8] = b":mock";
@@ -40,12 +42,14 @@ fn should_return_storage() {
 	let mut core = tokio::runtime::Runtime::new().unwrap();
 	let client = TestClientBuilder::new()
 		.add_extra_storage(KEY.to_vec(), VALUE.to_vec())
-		.add_extra_child_storage(STORAGE_KEY.to_vec(), KEY.to_vec(), CHILD_VALUE.to_vec())
+		.add_extra_child_storage(STORAGE_KEY.to_vec(), CHILD_INFO, KEY.to_vec(), CHILD_VALUE.to_vec())
 		.build();
 	let genesis_hash = client.genesis_hash();
 	let client = new_full(Arc::new(client), Subscriptions::new(Arc::new(core.executor())));
 	let key = StorageKey(KEY.to_vec());
 	let storage_key = StorageKey(STORAGE_KEY.to_vec());
+	let (child_info, child_type) = CHILD_INFO.info();
+	let child_info = StorageKey(child_info.to_vec());
 
 	assert_eq!(
 		client.storage(key.clone(), Some(genesis_hash).into()).wait()
@@ -63,7 +67,7 @@ fn should_return_storage() {
 	);
 	assert_eq!(
 		core.block_on(
-			client.child_storage(storage_key, key, Some(genesis_hash).into())
+			client.child_storage(storage_key, child_info, child_type, key, Some(genesis_hash).into())
 				.map(|x| x.map(|x| x.0.len()))
 		).unwrap().unwrap() as usize,
 		CHILD_VALUE.len(),
@@ -73,27 +77,48 @@ fn should_return_storage() {
 
 #[test]
 fn should_return_child_storage() {
+	let (child_info, child_type) = CHILD_INFO.info();
+	let child_info = StorageKey(child_info.to_vec());
 	let core = tokio::runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::TestClientBuilder::new()
-		.add_child_storage("test", "key", vec![42_u8])
+		.add_child_storage("test", "key", CHILD_INFO, vec![42_u8])
 		.build());
 	let genesis_hash = client.genesis_hash();
 	let client = new_full(client, Subscriptions::new(Arc::new(core.executor())));
-	let child_key = StorageKey(well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect());
+	let child_key = StorageKey(
+		well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect()
+	);
 	let key = StorageKey(b"key".to_vec());
 
 
 	assert_matches!(
-		client.child_storage(child_key.clone(), key.clone(), Some(genesis_hash).into()).wait(),
+		client.child_storage(
+			child_key.clone(),
+			child_info.clone(),
+			child_type,
+			key.clone(),
+			Some(genesis_hash).into(),
+		).wait(),
 		Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1
 	);
 	assert_matches!(
-		client.child_storage_hash(child_key.clone(), key.clone(), Some(genesis_hash).into())
-			.wait().map(|x| x.is_some()),
+		client.child_storage_hash(
+			child_key.clone(),
+			child_info.clone(),
+			child_type,
+			key.clone(),
+			Some(genesis_hash).into(),
+		).wait().map(|x| x.is_some()),
 		Ok(true)
 	);
 	assert_matches!(
-		client.child_storage_size(child_key.clone(), key.clone(), None).wait(),
+		client.child_storage_size(
+			child_key.clone(),
+			child_info.clone(),
+			child_type,
+			key.clone(),
+			None,
+		).wait(),
 		Ok(Some(1))
 	);
 }