best_containing operations (issue 603) (#740)

* add stub for Client.best_chain_containing_block_hash

* add fn blockchain::Backend::leaf_hashes

* fix typo

* sketch out Client.best_chain_containing_block_hash

* fix indent

* Blockchain.leaf_hashes -> Blockchain.leaves

* add unimplemented! stub impls for Blockchain.leaves

* start impl of Blockchain.leaves for in-memory client db

* Client.best_chain_containing...: check canonical first and make compile

* first rough attempt at maintaining leaf list in in-memory db

* fix tab indent

* add test best_chain_containing_single_block

* add failing test best_chain_containing_with_fork

* pub use client::blockchain; in test-client to prevent circular dep in client tests

* best_chain_containing_with_single_block: improve and test leaves

* far improve in-memory Backend::leaves impl

* test blockchain::Backend::leaves more thoroughly

* handle more edge cases in blockchain::Backend::leaves impl for in memory

* fix test best_chain_containing_with_fork (two distinct test blocks had same hash)

* make best_chain_containing_block_hash pass existing tests

* improve docstring for Blockchain::leaves

* Client.best_chain_containing: some cleanup. support max_block_number

* best_chain_containing: remove broken outcommented fast check for best = canonical

* remove blank line

* best_block_containing: return None if target_hash not found

* best_chain_containing: add unreachable! at end of function

* improve tests for best_chain_containing

* renames

* more elaborate test scenario for best_containing

* best_containing: fix restriction of search through maybe_max_number

* best_containing: tests for restriction of search

* get rid of unnecessary clones

* replace Client::new_in_mem by new_with_backend which is useful for testing backends

* add test_client::new_with_backend for testing different backend impls

* add test for in_mem::Backend::leaves

* remove unused imports

* in_mem test_leaves: simplify

* flesh out tests for in_mem leaves impl

* remove tests for leaves from client which are now covered in implementing module

* improve comment

* add Client.new_in_mem again

* unwrap in test_client::new_with_backend

* make test_client::BlockBuilderExt work not just with in-mem backend

* make test client ext not just work with in mem backend

* add failing Backend.leaves test for client-db

* update Cargo.lock

* replace KeccakHasher with Blake2Hasher

* refactor

address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217822862

* refactor using NumberFor

address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217823341

* add test that exposes possible problem

* update docstring for Client.best_containing

* extract test for Backend.leaves for reuse

* improve test blockchain_header_and_hash_return_blocks_from_canonical_chain_given_block_numbers

* extract test_blockchain_query_by_number_gets_canonical to easily test multiple impls

* remove whitespace

* remove todo

* Client.best_containing: pre-empt search loop when target in canonical

* best_containing: prevent race condition by holding import lock

* add todo

* extract leaf list update code into function

* add comment

* client-db: use in-memory-kvdb for tests

* use BTreeSet to store leaves for in-mem which is faster and simpler

* add docstring

* add comments and fix formatting

* add initial raw version of LeafSet

* remove Client::update_leaves which has been superceded by LeafSet

* use LeafSet in in-mem backend

* keccak -> blake2

* don't reexport codec traits in primitives

addresses https://github.com/paritytech/substrate/pull/740#discussion_r219538185

* fix rebase mistake

* improve LeafSet and use it in state-db

* correct Transfer nonces to fix ApplyExtinsicFailed(Stale)

* use given backend in canoncal test

* kill dead tree-route code in util

* fix warnings

* tests for leafset

* reorganizations in in_mem backend

* fix reorganization canon block logic

* DB commit and safe reversion on write error

* fix style nits

substrate/core/client/db/src/lib.rs

@@ -47,6 +47,9 @@ extern crate log;
 #[macro_use]
 extern crate parity_codec_derive;
 
+#[cfg(test)]
+extern crate substrate_test_client as test_client;
+
 #[cfg(test)]
 extern crate kvdb_memorydb;
 
@@ -74,6 +77,7 @@ use state_machine::backend::Backend as StateBackend;
 use executor::RuntimeInfo;
 use state_machine::{CodeExecutor, DBValue, ExecutionStrategy};
 use utils::{Meta, db_err, meta_keys, open_database, read_db, read_id, read_meta};
+use client::LeafSet;
 use state_db::StateDb;
 pub use state_db::PruningMode;
 
@@ -141,15 +145,18 @@ impl<'a> state_db::MetaDb for StateMetaDb<'a> {
 /// Block database
 pub struct BlockchainDb<Block: BlockT> {
 	db: Arc<KeyValueDB>,
-	meta: RwLock<Meta<<Block::Header as HeaderT>::Number, Block::Hash>>,
+	meta: RwLock<Meta<NumberFor<Block>, Block::Hash>>,
+	leaves: RwLock<LeafSet<Block::Hash, NumberFor<Block>>>,
 }
 
 impl<Block: BlockT> BlockchainDb<Block> {
 	fn new(db: Arc<KeyValueDB>) -> Result<Self, client::error::Error> {
 		let meta = read_meta::<Block>(&*db, columns::HEADER)?;
+		let leaves = LeafSet::read_from_db(&*db, columns::HEADER, meta_keys::LEAF_PREFIX)?;
 		Ok(BlockchainDb {
 			db,
-			meta: RwLock::new(meta)
+			leaves: RwLock::new(leaves),
+			meta: RwLock::new(meta),
 		})
 	}
 
@@ -249,6 +256,10 @@ impl<Block: BlockT> client::blockchain::Backend<Block> for BlockchainDb<Block> {
 	fn cache(&self) -> Option<&client::blockchain::Cache<Block>> {
 		None
 	}
+
+	fn leaves(&self) -> Result<Vec<Block::Hash>, client::error::Error> {
+		Ok(self.leaves.read().hashes())
+	}
 }
 
 /// Database transaction
@@ -534,6 +545,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 		let mut transaction = DBTransaction::new();
 		if let Some(pending_block) = operation.pending_block {
 			let hash = pending_block.header.hash();
+			let parent_hash = *pending_block.header.parent_hash();
 			let number = pending_block.header.number().clone();
 
 			transaction.put(columns::HEADER, hash.as_ref(), &pending_block.header.encode());
@@ -549,7 +561,6 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 
 				// cannot find tree route with empty DB.
 				if meta.best_hash != Default::default() {
-					let parent_hash = *pending_block.header.parent_hash();
 					let tree_route = ::client::blockchain::tree_route(
 						&self.blockchain,
 						BlockId::Hash(meta.best_hash),
@@ -624,10 +635,25 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 			debug!(target: "db", "DB Commit {:?} ({}), best = {}", hash, number,
 				pending_block.leaf_state.is_best());
 
-			self.storage.db.write(transaction).map_err(db_err)?;
+			{
+				let mut leaves = self.blockchain.leaves.write();
+				let displaced_leaf = leaves.import(hash, number, parent_hash);
+				leaves.prepare_transaction(&mut transaction, columns::HEADER, meta_keys::LEAF_PREFIX);
+
+				let write_result = self.storage.db.write(transaction).map_err(db_err);
+				if let Err(e) = write_result {
+					// revert leaves set update, if there was one.
+					if let Some(displaced_leaf) = displaced_leaf {
+						leaves.undo(displaced_leaf);
+					}
+					return Err(e);
+				}
+				drop(leaves);
+			}
+
 			self.blockchain.update_meta(
-				hash,
-				number,
+				hash.clone(),
+				number.clone(),
 				pending_block.leaf_state.is_best(),
 				finalized,
 			);
@@ -668,14 +694,15 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 					apply_state_commit(&mut transaction, commit);
 					let removed = best.clone();
 					best -= As::sa(1);
-					let hash = self.blockchain.hash(best)?.ok_or_else(
+					let header = self.blockchain.header(BlockId::Number(best))?.ok_or_else(
 						|| client::error::ErrorKind::UnknownBlock(
-							format!("Error reverting to {}. Block hash not found.", best)))?;
+							format!("Error reverting to {}. Block header not found.", best)))?;
 
-					transaction.put(columns::META, meta_keys::BEST_BLOCK, hash.as_ref());
+					transaction.put(columns::META, meta_keys::BEST_BLOCK, header.hash().as_ref());
 					transaction.delete(columns::HASH_LOOKUP, &::utils::number_to_lookup_key(removed));
 					self.storage.db.write(transaction).map_err(db_err)?;
-					self.blockchain.update_meta(hash, best, true, false);
+					self.blockchain.update_meta(header.hash().clone(), best.clone(), true, false);
+					self.blockchain.leaves.write().revert(header.hash().clone(), header.number().clone(), header.parent_hash().clone());
 				}
 				None => return Ok(As::sa(c))
 			}
@@ -723,6 +750,7 @@ mod tests {
 	use client::blockchain::HeaderBackend as BlockchainHeaderBackend;
 	use runtime_primitives::testing::{Header, Block as RawBlock};
 	use state_machine::{TrieMut, TrieDBMut, ChangesTrieStorage};
+	use test_client;
 
 	type Block = RawBlock<u64>;
 
@@ -846,7 +874,7 @@ mod tests {
 
 			op.reset_storage(storage.iter().cloned()).unwrap();
 			op.set_block_data(
-				header,
+				header.clone(),
 				Some(vec![]),
 				None,
 				NewBlockState::Best,
@@ -1101,4 +1129,18 @@ mod tests {
 			assert!(tree_route.enacted().is_empty());
 		}
 	}
+
+	#[test]
+	fn test_leaves_with_complex_block_tree() {
+		let backend: Arc<Backend<test_client::runtime::Block>> = Arc::new(Backend::new_test(20));
+
+		test_client::trait_tests::test_leaves_for_backend(backend);
+	}
+
+	#[test]
+	fn test_blockchain_query_by_number_gets_canonical() {
+		let backend: Arc<Backend<test_client::runtime::Block>> = Arc::new(Backend::new_test(20));
+
+		test_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend);
+	}
 }

substrate/core/client/db/src/light.rs

@@ -24,7 +24,7 @@ use kvdb::{KeyValueDB, DBTransaction};
 use client::backend::NewBlockState;
 use client::blockchain::{BlockStatus, Cache as BlockchainCache,
 	HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo};
-use client::cht;
+use client::{cht, LeafSet};
 use client::error::{ErrorKind as ClientErrorKind, Result as ClientResult};
 use client::light::blockchain::Storage as LightBlockchainStorage;
 use codec::{Decode, Encode};
@@ -53,6 +53,7 @@ pub(crate) const AUTHORITIES_ENTRIES_TO_KEEP: u64 = cht::SIZE;
 pub struct LightStorage<Block: BlockT> {
 	db: Arc<KeyValueDB>,
 	meta: RwLock<Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>>,
+	leaves: RwLock<LeafSet<Block::Hash, NumberFor<Block>>>,
 	_cache: DbCache<Block>,
 }
 
@@ -92,10 +93,12 @@ impl<Block> LightStorage<Block>
 			columns::AUTHORITIES
 		)?;
 		let meta = RwLock::new(read_meta::<Block>(&*db, columns::HEADER)?);
+		let leaves = RwLock::new(LeafSet::read_from_db(&*db, columns::HEADER, meta_keys::LEAF_PREFIX)?);
 
 		Ok(LightStorage {
 			db,
 			meta,
+			leaves,
 			_cache: cache,
 		})
 	}
@@ -239,6 +242,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 
 		let hash = header.hash();
 		let number = *header.number();
+		let parent_hash = *header.parent_hash();
 
 		transaction.put(columns::HEADER, hash.as_ref(), &header.encode());
 		transaction.put(columns::HASH_LOOKUP, &number_to_lookup_key(number), hash.as_ref());
@@ -250,7 +254,6 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 			{
 				let meta = self.meta.read();
 				if meta.best_hash != Default::default() {
-					let parent_hash = *header.parent_hash();
 					let tree_route = ::client::blockchain::tree_route(
 						self,
 						BlockId::Hash(meta.best_hash),
@@ -294,8 +297,20 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
 			self.note_finalized(&mut transaction, &header, hash)?;
 		}
 
-		debug!("Light DB Commit {:?} ({})", hash, number);
-		self.db.write(transaction).map_err(db_err)?;
+		{
+			let mut leaves = self.leaves.write();
+			let displaced_leaf = leaves.import(hash, number, parent_hash);
+
+			debug!("Light DB Commit {:?} ({})", hash, number);
+			let write_result = self.db.write(transaction).map_err(db_err);
+			if let Err(e) = write_result {
+				// revert leaves set update if there was one.
+				if let Some(displaced_leaf) = displaced_leaf {
+					leaves.undo(displaced_leaf);
+				}
+				return Err(e);
+			}
+		}
 		self.update_meta(hash, number, leaf_state.is_best(), finalized);
 
 		Ok(())

substrate/core/client/db/src/utils.rs

@@ -48,6 +48,8 @@ pub mod meta_keys {
 	pub const BEST_AUTHORITIES: &[u8; 4] = b"auth";
 	/// Genesis block hash.
 	pub const GENESIS_HASH: &[u8; 3] = b"gen";
+	/// Leaves prefix list key.
+	pub const LEAF_PREFIX: &[u8; 4] = b"leaf";
 }
 
 /// Database metadata.