new state api state_getKeys to expose storage keys (#1380)

substrate/core/client/db/src/storage_cache.rs

@@ -346,6 +346,10 @@ impl<H: Hasher, S: StateBackend<H>, B:Block> StateBackend<H> for CachingState<H,
 		self.state.pairs()
 	}
 
+	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+		self.state.keys(prefix)
+	}
+
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
 		self.state.try_into_trie_backend()
 	}

substrate/core/client/src/client.rs

@@ -265,6 +265,12 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		&self.backend
 	}
 
+	/// Return storage entry keys in state in a block of given hash with given prefix.
+	pub fn storage_keys(&self, id: &BlockId<Block>, key_prefix: &StorageKey) -> error::Result<Vec<StorageKey>> {
+		let keys = self.state_at(id)?.keys(&key_prefix.0).into_iter().map(StorageKey).collect();
+		Ok(keys)
+	}
+
 	/// Return single storage entry of contract under given address in state in a block of given hash.
 	pub fn storage(&self, id: &BlockId<Block>, key: &StorageKey) -> error::Result<Option<StorageData>> {
 		Ok(self.state_at(id)?

substrate/core/client/src/light/backend.rs

@@ -279,6 +279,11 @@ where
 		Vec::new()
 	}
 
+	fn keys(&self, _prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+		// whole state is not available on light node
+		Vec::new()
+	}
+
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
 		None
 	}

substrate/core/rpc/src/state/mod.rs

@@ -51,6 +51,10 @@ build_rpc_trait! {
 		#[rpc(name = "state_call", alias = ["state_callAt", ])]
 		fn call(&self, String, Bytes, Trailing<Hash>) -> Result<Bytes>;
 
+		/// Returns the keys with prefix, leave empty to get all the keys
+		#[rpc(name = "state_getKeys")]
+		fn storage_keys(&self, StorageKey, Trailing<Hash>) -> Result<Vec<StorageKey>>;
+
 		/// Returns a storage entry at a specific block's state.
 		#[rpc(name = "state_getStorage", alias = ["state_getStorageAt", ])]
 		fn storage(&self, StorageKey, Trailing<Hash>) -> Result<Option<StorageData>>;
@@ -148,6 +152,12 @@ impl<B, E, Block, RA> StateApi<Block::Hash> for State<B, E, Block, RA> where
 		Ok(Bytes(return_data))
 	}
 
+	fn storage_keys(&self, key_prefix: StorageKey, block: Trailing<Block::Hash>) -> Result<Vec<StorageKey>> {
+		let block = self.unwrap_or_best(block)?;
+		trace!(target: "rpc", "Querying storage keys at {:?}", block);
+		Ok(self.client.storage_keys(&BlockId::Hash(block), &key_prefix)?)
+	}
+
 	fn storage(&self, key: StorageKey, block: Trailing<Block::Hash>) -> Result<Option<StorageData>> {
 		let block = self.unwrap_or_best(block)?;
 		trace!(target: "rpc", "Querying storage at {:?} for key {}", block, HexDisplay::from(&key.0));

substrate/core/state-machine/src/backend.rs

@@ -86,6 +86,9 @@ pub trait Backend<H: Hasher> {
 	/// Get all key/value pairs into a Vec.
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
 
+	/// Get all keys with given prefix
+	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>>;
+
 	/// Try convert into trie backend.
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>>;
 }
@@ -283,6 +286,10 @@ impl<H: Hasher> Backend<H> for InMemory<H> where H::Out: HeapSizeOf {
 		self.inner.get(&None).into_iter().flat_map(|map| map.iter().map(|(k, v)| (k.clone(), v.clone()))).collect()
 	}
 
+	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+		self.inner.get(&None).into_iter().flat_map(|map| map.keys().filter(|k| k.starts_with(prefix)).cloned()).collect()
+	}
+
 	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
 		let mut mdb = MemoryDB::default();	// TODO: should be more correct and use ::new()
 		let mut root = None;

substrate/core/state-machine/src/proving_backend.rs

@@ -146,6 +146,10 @@ impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
 		self.backend.pairs()
 	}
 
+	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+		self.backend.keys(prefix)
+	}
+
 	fn storage_root<I>(&self, delta: I) -> (H::Out, MemoryDB<H>)
 		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
 	{

substrate/core/state-machine/src/trie_backend.rs

@@ -105,6 +105,26 @@ impl<S: TrieBackendStorage<H>, H: Hasher> Backend<H> for TrieBackend<S, H> where
 		}
 	}
 
+	fn keys(&self, prefix: &Vec<u8>) -> Vec<Vec<u8>> {
+		let mut read_overlay = MemoryDB::default();	// TODO: use new for correctness
+		let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
+
+		let collect_all = || -> Result<_, Box<TrieError<H::Out>>> {
+			let trie = TrieDB::<H>::new(&eph, self.essence.root())?;
+			let mut v = Vec::new();
+			for x in trie.iter()? {
+				let (key, _) = x?;
+				if key.starts_with(prefix) {
+					v.push(key.to_vec());
+				}
+			}
+
+			Ok(v)
+		};
+
+		collect_all().map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)).unwrap_or_default()
+	}
+
 	fn storage_root<I>(&self, delta: I) -> (H::Out, MemoryDB<H>)
 		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
 	{