From 98de97e1d956c132292d5753a36fa0d1baf2ee8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Sun, 19 May 2019 19:02:09 +0200 Subject: [PATCH] Access child storage over RPC. (#2586) * Access child storage over RPC. * Address review grumbles. * Test happy case in child_storage rpc. * Remove stray printlns * Fix line widths. * Bump runtime again. * Fix genesis storage root calculation for light clients. * Don't pass values to full_storage_root child_delta. --- substrate/core/client/db/src/storage_cache.rs | 6 +- substrate/core/client/src/client.rs | 31 ++++++- substrate/core/client/src/light/backend.rs | 15 +++- substrate/core/rpc/src/state/mod.rs | 87 ++++++++++++++++++- substrate/core/rpc/src/state/tests.rs | 44 +++++++++- substrate/core/sr-primitives/src/lib.rs | 22 ++++- substrate/core/state-machine/src/backend.rs | 44 ++++++++-- .../core/state-machine/src/proving_backend.rs | 6 +- .../core/state-machine/src/trie_backend.rs | 2 +- substrate/core/test-client/src/lib.rs | 14 ++- .../node-template/runtime/wasm/Cargo.lock | 1 + 11 files changed, 248 insertions(+), 24 deletions(-) diff --git a/substrate/core/client/db/src/storage_cache.rs b/substrate/core/client/db/src/storage_cache.rs index 439a749c85..bc0a179cf8 100644 --- a/substrate/core/client/db/src/storage_cache.rs +++ b/substrate/core/client/db/src/storage_cache.rs @@ -397,10 +397,14 @@ impl, B:Block> StateBackend for CachingState) -> Vec> { + fn keys(&self, prefix: &[u8]) -> Vec> { self.state.keys(prefix) } + fn child_keys(&self, child_key: &[u8], prefix: &[u8]) -> Vec> { + self.state.child_keys(child_key, prefix) + } + fn try_into_trie_backend(self) -> Option> { self.state.try_into_trie_backend() } diff --git a/substrate/core/client/src/client.rs b/substrate/core/client/src/client.rs index 4091e23b5f..257f120d72 100644 --- a/substrate/core/client/src/client.rs +++ b/substrate/core/client/src/client.rs @@ -337,19 +337,46 @@ impl Client where self.import_lock.clone() } - /// Return storage entry keys in state in a block of given hash with given prefix. + /// Given a `BlockId` and a key prefix, return the matching child storage keys in that block. pub fn storage_keys(&self, id: &BlockId, key_prefix: &StorageKey) -> error::Result> { let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect(); Ok(keys) } - /// Return single storage entry of contract under given address in state in a block of given hash. + /// Given a `BlockId` and a key, return the value under the key in that block. pub fn storage(&self, id: &BlockId, key: &StorageKey) -> error::Result> { Ok(self.state_at(id)? .storage(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))? .map(StorageData)) } + /// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys. + pub fn child_storage_keys( + &self, + id: &BlockId, + child_storage_key: &StorageKey, + key_prefix: &StorageKey + ) -> error::Result> { + let keys = self.state_at(id)? + .child_keys(&child_storage_key.0, &key_prefix.0) + .into_iter() + .map(StorageKey) + .collect(); + Ok(keys) + } + + /// Given a `BlockId`, a key and a child storage key, return the value under the key in that block. + pub fn child_storage( + &self, + id: &BlockId, + child_storage_key: &StorageKey, + key: &StorageKey + ) -> error::Result> { + Ok(self.state_at(id)? + .child_storage(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))? + .map(StorageData)) + } + /// Get the code at a given block. pub fn code_at(&self, id: &BlockId) -> error::Result> { Ok(self.storage(id, &StorageKey(well_known_keys::CODE.to_vec()))? diff --git a/substrate/core/client/src/light/backend.rs b/substrate/core/client/src/light/backend.rs index 52cdb6a626..0579c8c8d4 100644 --- a/substrate/core/client/src/light/backend.rs +++ b/substrate/core/client/src/light/backend.rs @@ -278,11 +278,20 @@ where // this is only called when genesis block is imported => shouldn't be performance bottleneck let mut storage: HashMap>, StorageOverlay> = HashMap::new(); storage.insert(None, top); + + // create a list of children keys to re-compute roots for + let child_delta = children.keys() + .cloned() + .map(|storage_key| (storage_key, None)) + .collect::>(); + + // make sure to persist the child storage for (child_key, child_storage) in children { storage.insert(Some(child_key), child_storage); } + let storage_update: InMemoryState = storage.into(); - let (storage_root, _) = storage_update.storage_root(::std::iter::empty()); + let (storage_root, _) = storage_update.full_storage_root(::std::iter::empty(), child_delta); self.storage_update = Some(storage_update); Ok(storage_root) @@ -373,7 +382,7 @@ where Vec::new() } - fn keys(&self, _prefix: &Vec) -> Vec> { + fn keys(&self, _prefix: &[u8]) -> Vec> { // whole state is not available on light node Vec::new() } @@ -465,7 +474,7 @@ where } } - fn keys(&self, prefix: &Vec) -> Vec> { + fn keys(&self, prefix: &[u8]) -> Vec> { match *self { OnDemandOrGenesisState::OnDemand(ref state) => StateBackend::::keys(state, prefix), diff --git a/substrate/core/rpc/src/state/mod.rs b/substrate/core/rpc/src/state/mod.rs index 168c0bd692..4cb7a3fbd8 100644 --- a/substrate/core/rpc/src/state/mod.rs +++ b/substrate/core/rpc/src/state/mod.rs @@ -57,7 +57,7 @@ pub trait StateApi { /// Returns the keys with prefix, leave empty to get all the keys #[rpc(name = "state_getKeys")] - fn storage_keys(&self, key: StorageKey, hash: Option) -> Result>; + fn storage_keys(&self, prefix: StorageKey, hash: Option) -> Result>; /// Returns a storage entry at a specific block's state. #[rpc(name = "state_getStorage", alias("state_getStorageAt"))] @@ -71,6 +71,40 @@ pub trait StateApi { #[rpc(name = "state_getStorageSize", alias("state_getStorageSizeAt"))] fn storage_size(&self, key: StorageKey, hash: Option) -> Result>; + /// Returns the keys with prefix from a child storage, leave empty to get all the keys + #[rpc(name = "state_getChildKeys")] + fn child_storage_keys( + &self, + child_storage_key: StorageKey, + prefix: StorageKey, + hash: Option + ) -> Result>; + + /// Returns a child storage entry at a specific block's state. + #[rpc(name = "state_getChildStorage")] + fn child_storage( + &self, + child_storage_key: StorageKey, + key: StorageKey, hash: Option + ) -> Result>; + + /// Returns the hash of a child storage entry at a block's state. + #[rpc(name = "state_getChildStorageHash")] + fn child_storage_hash( + &self, + child_storage_key: StorageKey, + key: StorageKey, hash: Option + ) -> Result>; + + /// Returns the size of a child storage entry at a block's state. + #[rpc(name = "state_getChildStorageSize")] + fn child_storage_size( + &self, + child_storage_key: StorageKey, + key: StorageKey, + hash: Option + ) -> Result>; + /// Returns the runtime metadata as an opaque blob. #[rpc(name = "state_getMetadata")] fn metadata(&self, hash: Option) -> Result; @@ -84,7 +118,12 @@ pub trait StateApi { /// NOTE This first returned result contains the initial state of storage for all keys. /// Subsequent values in the vector represent changes to the previous state (diffs). #[rpc(name = "state_queryStorage")] - fn query_storage(&self, keys: Vec, block: Hash, hash: Option) -> Result>>; + fn query_storage( + &self, + keys: Vec, + block: Hash, + hash: Option + ) -> Result>>; /// New runtime version subscription #[pubsub( @@ -324,6 +363,50 @@ impl StateApi for State where Ok(self.storage(key, block)?.map(|x| x.0.len() as u64)) } + fn child_storage( + &self, + child_storage_key: StorageKey, + key: StorageKey, + block: Option + ) -> Result> { + let block = self.unwrap_or_best(block)?; + trace!(target: "rpc", "Querying child storage at {:?} for key {}", block, HexDisplay::from(&key.0)); + Ok(self.client.child_storage(&BlockId::Hash(block), &child_storage_key, &key)?) + } + + fn child_storage_keys( + &self, + child_storage_key: StorageKey, + key_prefix: StorageKey, + block: Option + ) -> Result> { + let block = self.unwrap_or_best(block)?; + trace!(target: "rpc", "Querying child storage keys at {:?}", block); + Ok(self.client.child_storage_keys(&BlockId::Hash(block), &child_storage_key, &key_prefix)?) + } + + fn child_storage_hash( + &self, + child_storage_key: StorageKey, + key: StorageKey, + block: Option + ) -> Result> { + use runtime_primitives::traits::{Hash, Header as HeaderT}; + Ok( + self.child_storage(child_storage_key, key, block)? + .map(|x| ::Hashing::hash(&x.0)) + ) + } + + fn child_storage_size( + &self, + child_storage_key: StorageKey, + key: StorageKey, + block: Option + ) -> Result> { + Ok(self.child_storage(child_storage_key, key, block)?.map(|x| x.0.len() as u64)) + } + fn metadata(&self, block: Option) -> Result { let block = self.unwrap_or_best(block)?; self.client.runtime_api().metadata(&BlockId::Hash(block)).map(Into::into).map_err(Into::into) diff --git a/substrate/core/rpc/src/state/tests.rs b/substrate/core/rpc/src/state/tests.rs index 5321116c95..73a52fbc62 100644 --- a/substrate/core/rpc/src/state/tests.rs +++ b/substrate/core/rpc/src/state/tests.rs @@ -17,9 +17,10 @@ use super::*; use self::error::{Error, ErrorKind}; -use sr_io::blake2_256; use assert_matches::assert_matches; use consensus::BlockOrigin; +use primitives::storage::well_known_keys; +use sr_io::blake2_256; use test_client::{self, runtime, AccountKeyring, TestClient, BlockBuilderExt}; #[test] @@ -28,11 +29,46 @@ fn should_return_storage() { let client = Arc::new(test_client::new()); let genesis_hash = client.genesis_hash(); let client = State::new(client, Subscriptions::new(core.executor())); + let key = StorageKey(b":code".to_vec()); + + assert!( + client.storage(key.clone(), Some(genesis_hash).into()) + .map(|x| x.map(|x| x.0.len())).unwrap().unwrap() + > 195_000 + ); + assert_matches!( + client.storage_hash(key.clone(), Some(genesis_hash).into()).map(|x| x.is_some()), + Ok(true) + ); + assert!( + client.storage_size(key.clone(), None).unwrap().unwrap() + > 195_000 + ); +} + +#[test] +fn should_return_child_storage() { + let core = ::tokio::runtime::Runtime::new().unwrap(); + let client = Arc::new(test_client::new()); + let genesis_hash = client.genesis_hash(); + let client = State::new(client, Subscriptions::new(core.executor())); + let child_key = StorageKey(well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect()); + let key = StorageKey(b"key".to_vec()); + assert_matches!( - client.storage(StorageKey(vec![10]), Some(genesis_hash).into()), - Ok(None) - ) + client.child_storage(child_key.clone(), key.clone(), Some(genesis_hash).into()), + Ok(Some(StorageData(ref d))) if d[0] == 42 && d.len() == 1 + ); + assert_matches!( + client.child_storage_hash(child_key.clone(), key.clone(), Some(genesis_hash).into()) + .map(|x| x.is_some()), + Ok(true) + ); + assert_matches!( + client.child_storage_size(child_key.clone(), key.clone(), None), + Ok(Some(1)) + ); } #[test] diff --git a/substrate/core/sr-primitives/src/lib.rs b/substrate/core/sr-primitives/src/lib.rs index 65a4f3d822..bdecfe8981 100644 --- a/substrate/core/sr-primitives/src/lib.rs +++ b/substrate/core/sr-primitives/src/lib.rs @@ -102,12 +102,32 @@ impl BuildStorage for StorageOverlay { fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> { Ok((self, Default::default())) } - fn assimilate_storage(self, storage: &mut StorageOverlay, _child_storage: &mut ChildrenStorageOverlay) -> Result<(), String> { + fn assimilate_storage( + self, + storage: &mut StorageOverlay, + _child_storage: &mut ChildrenStorageOverlay + ) -> Result<(), String> { storage.extend(self); Ok(()) } } +#[cfg(feature = "std")] +impl BuildStorage for (StorageOverlay, ChildrenStorageOverlay) { + fn build_storage(self) -> Result<(StorageOverlay, ChildrenStorageOverlay), String> { + Ok(self) + } + fn assimilate_storage( + self, + storage: &mut StorageOverlay, + child_storage: &mut ChildrenStorageOverlay + )-> Result<(), String> { + storage.extend(self.0); + child_storage.extend(self.1); + Ok(()) + } +} + /// Consensus engine unique ID. pub type ConsensusEngineId = [u8; 4]; diff --git a/substrate/core/state-machine/src/backend.rs b/substrate/core/state-machine/src/backend.rs index 895a805e43..fd143a553e 100644 --- a/substrate/core/state-machine/src/backend.rs +++ b/substrate/core/state-machine/src/backend.rs @@ -88,7 +88,22 @@ pub trait Backend { fn pairs(&self) -> Vec<(Vec, Vec)>; /// Get all keys with given prefix - fn keys(&self, prefix: &Vec) -> Vec>; + fn keys(&self, prefix: &[u8]) -> Vec> { + let mut all = Vec::new(); + self.for_keys_with_prefix(prefix, |k| all.push(k.to_vec())); + all + } + + /// Get all keys of child storage with given prefix + fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { + let mut all = Vec::new(); + self.for_keys_in_child_storage(child_storage_key, |k| { + if k.starts_with(prefix) { + all.push(k.to_vec()); + } + }); + all + } /// Try convert into trie backend. fn try_into_trie_backend(self) -> Option>; @@ -286,7 +301,9 @@ impl Backend for InMemory { I: IntoIterator, Option>)>, ::Out: Ord, { - let existing_pairs = self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + let existing_pairs = self.inner.get(&None) + .into_iter() + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = trie_root::(existing_pairs.chain(transaction.iter().cloned()) @@ -307,7 +324,9 @@ impl Backend for InMemory { { let storage_key = storage_key.to_vec(); - let existing_pairs = self.inner.get(&Some(storage_key.clone())).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + let existing_pairs = self.inner.get(&Some(storage_key.clone())) + .into_iter() + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); let transaction: Vec<_> = delta.into_iter().collect(); let root = child_trie_root::( @@ -326,11 +345,24 @@ impl Backend for InMemory { } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))).collect() + self.inner.get(&None) + .into_iter() + .flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect() } - fn keys(&self, prefix: &Vec) -> Vec> { - self.inner.get(&None).into_iter().flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()).collect() + fn keys(&self, prefix: &[u8]) -> Vec> { + self.inner.get(&None) + .into_iter() + .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) + .collect() + } + + fn child_keys(&self, storage_key: &[u8], prefix: &[u8]) -> Vec> { + self.inner.get(&Some(storage_key.to_vec())) + .into_iter() + .flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()) + .collect() } fn try_into_trie_backend( diff --git a/substrate/core/state-machine/src/proving_backend.rs b/substrate/core/state-machine/src/proving_backend.rs index fa7e94f78f..c23838bc21 100644 --- a/substrate/core/state-machine/src/proving_backend.rs +++ b/substrate/core/state-machine/src/proving_backend.rs @@ -162,10 +162,14 @@ impl<'a, S, H> Backend for ProvingBackend<'a, S, H> self.backend.pairs() } - fn keys(&self, prefix: &Vec) -> Vec> { + fn keys(&self, prefix: &[u8]) -> Vec> { self.backend.keys(prefix) } + fn child_keys(&self, child_storage_key: &[u8], prefix: &[u8]) -> Vec> { + self.backend.child_keys(child_storage_key, prefix) + } + fn storage_root(&self, delta: I) -> (H::Out, Self::Transaction) where I: IntoIterator, Option>)> { diff --git a/substrate/core/state-machine/src/trie_backend.rs b/substrate/core/state-machine/src/trie_backend.rs index 1ce915d5f5..00c0aca006 100644 --- a/substrate/core/state-machine/src/trie_backend.rs +++ b/substrate/core/state-machine/src/trie_backend.rs @@ -105,7 +105,7 @@ impl, H: Hasher> Backend for TrieBackend where } } - fn keys(&self, prefix: &Vec) -> Vec> { + fn keys(&self, prefix: &[u8]) -> Vec> { let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay); diff --git a/substrate/core/test-client/src/lib.rs b/substrate/core/test-client/src/lib.rs index 770b3de323..5154fb0c1d 100644 --- a/substrate/core/test-client/src/lib.rs +++ b/substrate/core/test-client/src/lib.rs @@ -34,7 +34,8 @@ pub use keyring::{sr25519::Keyring as AuthorityKeyring, AccountKeyring}; use std::{sync::Arc, collections::HashMap}; use futures::future::FutureResult; use primitives::Blake2Hasher; -use runtime_primitives::StorageOverlay; +use primitives::storage::well_known_keys; +use runtime_primitives::{StorageOverlay, ChildrenStorageOverlay}; use runtime_primitives::traits::{ Block as BlockT, Header as HeaderT, Hash as HashT, NumberFor }; @@ -271,7 +272,7 @@ fn genesis_config(support_changes_trie: bool) -> GenesisConfig { fn genesis_storage( support_changes_trie: bool, extension: HashMap, Vec> -) -> StorageOverlay { +) -> (StorageOverlay, ChildrenStorageOverlay) { let mut storage = genesis_config(support_changes_trie).genesis_map(); storage.extend(extension.into_iter()); @@ -280,7 +281,14 @@ fn genesis_storage( ); let block: runtime::Block = client::genesis::construct_genesis_block(state_root); storage.extend(additional_storage_with_genesis(&block)); - storage + + let mut child_storage = ChildrenStorageOverlay::default(); + child_storage.insert( + well_known_keys::CHILD_STORAGE_KEY_PREFIX.iter().chain(b"test").cloned().collect(), + vec![(b"key".to_vec(), vec![42_u8])].into_iter().collect() + ); + + (storage, child_storage) } impl client::light::fetcher::Fetcher for LightFetcher { diff --git a/substrate/node-template/runtime/wasm/Cargo.lock b/substrate/node-template/runtime/wasm/Cargo.lock index bd69f55c62..cfbf85fbf2 100644 --- a/substrate/node-template/runtime/wasm/Cargo.lock +++ b/substrate/node-template/runtime/wasm/Cargo.lock @@ -2359,6 +2359,7 @@ version = "2.0.0" dependencies = [ "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-io 2.0.0", "sr-primitives 2.0.0", "sr-std 2.0.0", "srml-support 2.0.0",