Access child storage over RPC. (#2586)

* Access child storage over RPC.

* Address review grumbles.

* Test happy case in child_storage rpc.

* Remove stray printlns

* Fix line widths.

* Bump runtime again.

* Fix genesis storage root calculation for light clients.

* Don't pass values to full_storage_root child_delta.

substrate/core/client/db/src/storage_cache.rs

@@ -397,10 +397,14 @@ impl<H: Hasher, S: StateBackend<H>, B:Block> StateBackend<H> for CachingState<H,
 		self.state.pairs()
 	}
 
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
 		self.state.keys(prefix)
 	}
 
+	fn child_keys(&self, child_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+		self.state.child_keys(child_key, prefix)
+	}
+
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
 		self.state.try_into_trie_backend()
 	}

substrate/core/client/src/client.rs

@@ -337,19 +337,46 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		self.import_lock.clone()
 	}
 
-	/// Return storage entry keys in state in a block of given hash with given prefix.
+	/// Given a `BlockId` and a key prefix, return the matching child storage keys in that block.
 	pub fn storage_keys(&self, id: &BlockId<Block>, key_prefix: &StorageKey) -> error::Result<Vec<StorageKey>> {
 		let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect();
 		Ok(keys)
 	}
 
-	/// Return single storage entry of contract under given address in state in a block of given hash.
+	/// Given a `BlockId` and a key, return the value under the key in that block.
 	pub fn storage(&self, id: &BlockId<Block>, key: &StorageKey) -> error::Result<Option<StorageData>> {
 		Ok(self.state_at(id)?
 			.storage(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
 			.map(StorageData))
 	}
 
+	/// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys.
+	pub fn child_storage_keys(
+		&self,
+		id: &BlockId<Block>,
+		child_storage_key: &StorageKey,
+		key_prefix: &StorageKey
+	) -> error::Result<Vec<StorageKey>> {
+		let keys = self.state_at(id)?
+			.child_keys(&child_storage_key.0, &key_prefix.0)
+			.into_iter()
+			.map(StorageKey)
+			.collect();
+		Ok(keys)
+	}
+
+	/// Given a `BlockId`, a key and a child storage key, return the value under the key in that block.
+	pub fn child_storage(
+		&self,
+		id: &BlockId<Block>,
+		child_storage_key: &StorageKey,
+		key: &StorageKey
+	) -> error::Result<Option<StorageData>> {
+		Ok(self.state_at(id)?
+			.child_storage(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
+			.map(StorageData))
+	}
+
 	/// Get the code at a given block.
 	pub fn code_at(&self, id: &BlockId<Block>) -> error::Result<Vec<u8>> {
 		Ok(self.storage(id, &StorageKey(well_known_keys::CODE.to_vec()))?

substrate/core/client/src/light/backend.rs

@@ -278,11 +278,20 @@ where
 		// this is only called when genesis block is imported => shouldn't be performance bottleneck
 		let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
 		storage.insert(None, top);
+
+		// create a list of children keys to re-compute roots for
+		let child_delta = children.keys()
+			.cloned()
+			.map(|storage_key| (storage_key, None))
+			.collect::<Vec<_>>();
+
+		// make sure to persist the child storage
 		for (child_key, child_storage) in children {
 			storage.insert(Some(child_key), child_storage);
 		}
+
 		let storage_update: InMemoryState<H> = storage.into();
-		let (storage_root, _) = storage_update.storage_root(::std::iter::empty());
+		let (storage_root, _) = storage_update.full_storage_root(::std::iter::empty(), child_delta);
 		self.storage_update = Some(storage_update);
 
 		Ok(storage_root)
@@ -373,7 +382,7 @@ where
 		Vec::new()
 	}
 
-	fn keys(&self, _prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+	fn keys(&self, _prefix: &[u8]) -> Vec<Vec<u8>> {
 		// whole state is not available on light node
 		Vec::new()
 	}
@@ -465,7 +474,7 @@ where
 		}
 	}
 
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
 		match *self {
 			OnDemandOrGenesisState::OnDemand(ref state) =>
 				StateBackend::<H>::keys(state, prefix),

substrate/core/rpc/src/state/mod.rs

@@ -57,7 +57,7 @@ pub trait StateApi<Hash> {
 
 	/// Returns the keys with prefix, leave empty to get all the keys
 	#[rpc(name = "state_getKeys")]
-	fn storage_keys(&self, key: StorageKey, hash: Option<Hash>) -> Result<Vec<StorageKey>>;
+	fn storage_keys(&self, prefix: StorageKey, hash: Option<Hash>) -> Result<Vec<StorageKey>>;
 
 	/// Returns a storage entry at a specific block's state.
 	#[rpc(name = "state_getStorage", alias("state_getStorageAt"))]
@@ -71,6 +71,40 @@ pub trait StateApi<Hash> {
 	#[rpc(name = "state_getStorageSize", alias("state_getStorageSizeAt"))]
 	fn storage_size(&self, key: StorageKey, hash: Option<Hash>) -> Result<Option<u64>>;
 
+	/// Returns the keys with prefix from a child storage, leave empty to get all the keys
+	#[rpc(name = "state_getChildKeys")]
+	fn child_storage_keys(
+		&self,
+		child_storage_key: StorageKey,
+		prefix: StorageKey,
+		hash: Option<Hash>
+	) -> Result<Vec<StorageKey>>;
+
+	/// Returns a child storage entry at a specific block's state.
+	#[rpc(name = "state_getChildStorage")]
+	fn child_storage(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey, hash: Option<Hash>
+	) -> Result<Option<StorageData>>;
+
+	/// Returns the hash of a child storage entry at a block's state.
+	#[rpc(name = "state_getChildStorageHash")]
+	fn child_storage_hash(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey, hash: Option<Hash>
+	) -> Result<Option<Hash>>;
+
+	/// Returns the size of a child storage entry at a block's state.
+	#[rpc(name = "state_getChildStorageSize")]
+	fn child_storage_size(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey,
+		hash: Option<Hash>
+	) -> Result<Option<u64>>;
+
 	/// Returns the runtime metadata as an opaque blob.
 	#[rpc(name = "state_getMetadata")]
 	fn metadata(&self, hash: Option<Hash>) -> Result<Bytes>;
@@ -84,7 +118,12 @@ pub trait StateApi<Hash> {
 	/// NOTE This first returned result contains the initial state of storage for all keys.
 	/// Subsequent values in the vector represent changes to the previous state (diffs).
 	#[rpc(name = "state_queryStorage")]
-	fn query_storage(&self, keys: Vec<StorageKey>, block: Hash, hash: Option<Hash>) -> Result<Vec<StorageChangeSet<Hash>>>;
+	fn query_storage(
+		&self,
+		keys: Vec<StorageKey>,
+		block: Hash,
+		hash: Option<Hash>
+	) -> Result<Vec<StorageChangeSet<Hash>>>;
 
 	/// New runtime version subscription
 	#[pubsub(
@@ -324,6 +363,50 @@ impl<B, E, Block, RA> StateApi<Block::Hash> for State<B, E, Block, RA> where
 		Ok(self.storage(key, block)?.map(|x| x.0.len() as u64))
 	}
 
+	fn child_storage(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey,
+		block: Option<Block::Hash>
+	) -> Result<Option<StorageData>> {
+		let block = self.unwrap_or_best(block)?;
+		trace!(target: "rpc", "Querying child storage at {:?} for key {}", block, HexDisplay::from(&key.0));
+		Ok(self.client.child_storage(&BlockId::Hash(block), &child_storage_key, &key)?)
+	}
+
+	fn child_storage_keys(
+		&self,
+		child_storage_key: StorageKey,
+		key_prefix: StorageKey,
+		block: Option<Block::Hash>
+	) -> Result<Vec<StorageKey>> {
+		let block = self.unwrap_or_best(block)?;
+		trace!(target: "rpc", "Querying child storage keys at {:?}", block);
+		Ok(self.client.child_storage_keys(&BlockId::Hash(block), &child_storage_key, &key_prefix)?)
+	}
+
+	fn child_storage_hash(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey,
+		block: Option<Block::Hash>
+	) -> Result<Option<Block::Hash>> {
+		use runtime_primitives::traits::{Hash, Header as HeaderT};
+		Ok(
+			self.child_storage(child_storage_key, key, block)?
+				.map(|x| <Block::Header as HeaderT>::Hashing::hash(&x.0))
+		)
+	}
+
+	fn child_storage_size(
+		&self,
+		child_storage_key: StorageKey,
+		key: StorageKey,
+		block: Option<Block::Hash>
+	) -> Result<Option<u64>> {
+		Ok(self.child_storage(child_storage_key, key, block)?.map(|x| x.0.len() as u64))
+	}
+
 	fn metadata(&self, block: Option<Block::Hash>) -> Result<Bytes> {
 		let block = self.unwrap_or_best(block)?;
 		self.client.runtime_api().metadata(&BlockId::Hash(block)).map(Into::into).map_err(Into::into)

substrate/core/rpc/src/state/tests.rs

@@ -17,9 +17,10 @@
 use super::*;
 use self::error::{Error, ErrorKind};
 
-use sr_io::blake2_256;
 use assert_matches::assert_matches;
 use consensus::BlockOrigin;
+use primitives::storage::well_known_keys;
+use sr_io::blake2_256;
 use test_client::{self, runtime, AccountKeyring, TestClient, BlockBuilderExt};
 
 #[test]
@@ -28,11 +29,46 @@ fn should_return_storage() {
 	let client = Arc::new(test_client::new());
 	let genesis_hash = client.genesis_hash();
 	let client = State::new(client, Subscriptions::new(core.executor()));
+	let key = StorageKey(b":code".to_vec());
+
+	assert!(
+		client.storage(key.clone(), Some(genesis_hash).into())
+			.map(|x| x.map(|x| x.0.len())).unwrap().unwrap()
+		> 195_000
+	);
+	assert_matches!(
+		client.storage_hash(key.clone(), Some(genesis_hash).into()).map(|x| x.is_some()),
+		Ok(true)
+	);
+	assert!(
+		client.storage_size(key.clone(), None).unwrap().unwrap()
+		> 195_000
+	);
+}
+
+#[test]
+fn should_return_child_storage() {
+	let core = ::tokio::runtime::Runtime::new().unwrap();
+	let client = Arc::new(test_client::new());
+	let genesis_hash = client.genesis_hash();
+	let client = State::new(client, Subscriptions::new(core.executor()));
+	let child_key = StorageKey(well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect());
+	let key = StorageKey(b"key".to_vec());
+
 
 	assert_matches!(
-		client.storage(StorageKey(vec![10]), Some(genesis_hash).into()),
-		Ok(None)
-	)
+		client.child_storage(child_key.clone(), key.clone(), Some(genesis_hash).into()),
+		Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1
+	);
+	assert_matches!(
+		client.child_storage_hash(child_key.clone(), key.clone(), Some(genesis_hash).into())
+			.map(|x| x.is_some()),
+		Ok(true)
+	);
+	assert_matches!(
+		client.child_storage_size(child_key.clone(), key.clone(), None),
+		Ok(Some(1))
+	);
 }
 
 #[test]

substrate/core/sr-primitives/src/lib.rs

@@ -102,12 +102,32 @@ impl BuildStorage for StorageOverlay {
 	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
 		Ok((self, Default::default()))
 	}
-	fn assimilate_storage(self, storage: &mut StorageOverlay, _child_storage: &mut ChildrenStorageOverlay) -> Result<(), String> {
+	fn assimilate_storage(
+		self,
+		storage: &mut StorageOverlay,
+		_child_storage: &mut ChildrenStorageOverlay
+	) -> Result<(), String> {
 		storage.extend(self);
 		Ok(())
 	}
 }
 
+#[cfg(feature = "std")]
+impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) {
+	fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> {
+		Ok(self)
+	}
+	fn assimilate_storage(
+		self,
+		storage: &mut StorageOverlay,
+		child_storage: &mut ChildrenStorageOverlay
+	)-> Result<(), String> {
+		storage.extend(self.0);
+		child_storage.extend(self.1);
+		Ok(())
+	}
+}
+
 /// Consensus engine unique ID.
 pub type ConsensusEngineId = [u8; 4];
 

substrate/core/state-machine/src/backend.rs

@@ -88,7 +88,22 @@ pub trait Backend<H: Hasher> {
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
 
 	/// Get all keys with given prefix
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>>;
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+		let mut all = Vec::new();
+		self.for_keys_with_prefix(prefix, |k| all.push(k.to_vec()));
+		all
+	}
+
+	/// Get all keys of child storage with given prefix
+	fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+		let mut all = Vec::new();
+		self.for_keys_in_child_storage(child_storage_key, |k| {
+			if k.starts_with(prefix) {
+				all.push(k.to_vec());
+			}
+		});
+		all
+	}
 
 	/// Try convert into trie backend.
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>>;
@@ -286,7 +301,9 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
 		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
 		<H as Hasher>::Out: Ord,
 	{
-		let existing_pairs = self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+		let existing_pairs = self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
 
 		let transaction: Vec<_> = delta.into_iter().collect();
 		let root = trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
@@ -307,7 +324,9 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
 	{
 		let storage_key = storage_key.to_vec();
 
-		let existing_pairs = self.inner.get(&Some(storage_key.clone())).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
+		let existing_pairs = self.inner.get(&Some(storage_key.clone()))
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone()))));
 
 		let transaction: Vec<_> = delta.into_iter().collect();
 		let root = child_trie_root::<H, _, _, _>(
@@ -326,11 +345,24 @@ impl<H: Hasher> Backend<H> for InMemory<H> {
 	}
 
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
-		self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))).collect()
+		self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone())))
+			.collect()
 	}
 
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
-		self.inner.get(&None).into_iter().flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()).collect()
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+		self.inner.get(&None)
+			.into_iter()
+			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+			.collect()
+	}
+
+	fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+		self.inner.get(&Some(storage_key.to_vec()))
+			.into_iter()
+			.flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned())
+			.collect()
 	}
 
 	fn try_into_trie_backend(

substrate/core/state-machine/src/proving_backend.rs

@@ -162,10 +162,14 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
 		self.backend.pairs()
 	}
 
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
 		self.backend.keys(prefix)
 	}
 
+	fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec<Vec<u8>> {
+		self.backend.child_keys(child_storage_key, prefix)
+	}
+
 	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
 		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
 	{

substrate/core/state-machine/src/trie_backend.rs

@@ -105,7 +105,7 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
 		}
 	}
 
-	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
 		let mut read_overlay = S::Overlay::default();
 		let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
 

substrate/core/test-client/src/lib.rs

@@ -34,7 +34,8 @@ pub use keyring::{sr25519::Keyring as AuthorityKeyring, AccountKeyring};
 use std::{sync::Arc, collections::HashMap};
 use futures::future::FutureResult;
 use primitives::Blake2Hasher;
-use runtime_primitives::StorageOverlay;
+use primitives::storage::well_known_keys;
+use runtime_primitives::{StorageOverlay, ChildrenStorageOverlay};
 use runtime_primitives::traits::{
 	Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor
 };
@@ -271,7 +272,7 @@ fn genesis_config(support_changes_trie: bool) -> GenesisConfig {
 fn genesis_storage(
 	support_changes_trie: bool,
 	extension: HashMap<Vec<u8>, Vec<u8>>
-) -> StorageOverlay {
+) -> (StorageOverlay, ChildrenStorageOverlay) {
 	let mut storage = genesis_config(support_changes_trie).genesis_map();
 	storage.extend(extension.into_iter());
 
@@ -280,7 +281,14 @@ fn genesis_storage(
 	);
 	let block: runtime::Block = client::genesis::construct_genesis_block(state_root);
 	storage.extend(additional_storage_with_genesis(&block));
-	storage
+
+	let mut child_storage = ChildrenStorageOverlay::default();
+	child_storage.insert(
+		well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect(),
+		vec![(b"key".to_vec(), vec![42_u8])].into_iter().collect()
+	);
+
+	(storage, child_storage)
 }
 
 impl<Block: BlockT> client::light::fetcher::Fetcher<Block> for LightFetcher {

substrate/node-template/runtime/wasm/Cargo.lock

@@ -2359,6 +2359,7 @@ version = "2.0.0"
 dependencies = [
  "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sr-io 2.0.0",
  "sr-primitives 2.0.0",
  "sr-std 2.0.0",
  "srml-support 2.0.0",