best_containing operations (issue 603) (#740)

* add stub for Client.best_chain_containing_block_hash

* add fn blockchain::Backend::leaf_hashes

* fix typo

* sketch out Client.best_chain_containing_block_hash

* fix indent

* Blockchain.leaf_hashes -> Blockchain.leaves

* add unimplemented! stub impls for Blockchain.leaves

* start impl of Blockchain.leaves for in-memory client db

* Client.best_chain_containing...: check canonical first and make compile

* first rough attempt at maintaining leaf list in in-memory db

* fix tab indent

* add test best_chain_containing_single_block

* add failing test best_chain_containing_with_fork

* pub use client::blockchain; in test-client to prevent circular dep in client tests

* best_chain_containing_with_single_block: improve and test leaves

* far improve in-memory Backend::leaves impl

* test blockchain::Backend::leaves more thoroughly

* handle more edge cases in blockchain::Backend::leaves impl for in memory

* fix test best_chain_containing_with_fork (two distinct test blocks had same hash)

* make best_chain_containing_block_hash pass existing tests

* improve docstring for Blockchain::leaves

* Client.best_chain_containing: some cleanup. support max_block_number

* best_chain_containing: remove broken outcommented fast check for best = canonical

* remove blank line

* best_block_containing: return None if target_hash not found

* best_chain_containing: add unreachable! at end of function

* improve tests for best_chain_containing

* renames

* more elaborate test scenario for best_containing

* best_containing: fix restriction of search through maybe_max_number

* best_containing: tests for restriction of search

* get rid of unnecessary clones

* replace Client::new_in_mem by new_with_backend which is useful for testing backends

* add test_client::new_with_backend for testing different backend impls

* add test for in_mem::Backend::leaves

* remove unused imports

* in_mem test_leaves: simplify

* flesh out tests for in_mem leaves impl

* remove tests for leaves from client which are now covered in implementing module

* improve comment

* add Client.new_in_mem again

* unwrap in test_client::new_with_backend

* make test_client::BlockBuilderExt work not just with in-mem backend

* make test client ext not just work with in mem backend

* add failing Backend.leaves test for client-db

* update Cargo.lock

* replace KeccakHasher with Blake2Hasher

* refactor

address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217822862

* refactor using NumberFor

address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217823341

* add test that exposes possible problem

* update docstring for Client.best_containing

* extract test for Backend.leaves for reuse

* improve test blockchain_header_and_hash_return_blocks_from_canonical_chain_given_block_numbers

* extract test_blockchain_query_by_number_gets_canonical to easily test multiple impls

* remove whitespace

* remove todo

* Client.best_containing: pre-empt search loop when target in canonical

* best_containing: prevent race condition by holding import lock

* add todo

* extract leaf list update code into function

* add comment

* client-db: use in-memory-kvdb for tests

* use BTreeSet to store leaves for in-mem which is faster and simpler

* add docstring

* add comments and fix formatting

* add initial raw version of LeafSet

* remove Client::update_leaves which has been superceded by LeafSet

* use LeafSet in in-mem backend

* keccak -> blake2

* don't reexport codec traits in primitives

addresses https://github.com/paritytech/substrate/pull/740#discussion_r219538185

* fix rebase mistake

* improve LeafSet and use it in state-db

* correct Transfer nonces to fix ApplyExtinsicFailed(Stale)

* use given backend in canoncal test

* kill dead tree-route code in util

* fix warnings

* tests for leafset

* reorganizations in in_mem backend

* fix reorganization canon block logic

* DB commit and safe reversion on write error

* fix style nits

substrate/core/client/src/in_mem.rs

@@ -32,6 +32,7 @@ use state_machine::backend::{Backend as StateBackend, InMemory};
 use state_machine::InMemoryChangesTrieStorage;
 use hash_db::Hasher;
 use heapsize::HeapSizeOf;
+use leaves::LeafSet;
 use trie::MemoryDB;
 
 struct PendingBlock<B: BlockT> {
@@ -69,7 +70,7 @@ impl<B: BlockT> StoredBlock<B> {
 	fn extrinsics(&self) -> Option<&[B::Extrinsic]> {
 		match *self {
 			StoredBlock::Header(_, _) => None,
-			StoredBlock::Full(ref b, _) => Some(b.extrinsics())
+			StoredBlock::Full(ref b, _) => Some(b.extrinsics()),
 		}
 	}
 
@@ -93,6 +94,7 @@ struct BlockchainStorage<Block: BlockT> {
 	finalized_hash: Block::Hash,
 	genesis_hash: Block::Hash,
 	cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
+	leaves: LeafSet<Block::Hash, NumberFor<Block>>,
 }
 
 /// In-memory blockchain. Supports concurrent reads.
@@ -139,6 +141,7 @@ impl<Block: BlockT> Blockchain<Block> {
 				finalized_hash: Default::default(),
 				genesis_hash: Default::default(),
 				cht_roots: HashMap::new(),
+				leaves: LeafSet::new(),
 			}));
 		Blockchain {
 			storage: storage.clone(),
@@ -157,16 +160,50 @@ impl<Block: BlockT> Blockchain<Block> {
 		justification: Option<Justification<Block::Hash>>,
 		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
 		new_state: NewBlockState,
-	) {
+	) -> ::error::Result<()> {
 		let number = header.number().clone();
+		let best_tree_route = match new_state.is_best() {
+			false => None,
+			true => {
+				let best_hash = self.storage.read().best_hash;
+				if &best_hash == header.parent_hash() {
+					None
+				} else {
+					let route = ::blockchain::tree_route(
+						self,
+						BlockId::Hash(best_hash),
+						BlockId::Hash(*header.parent_hash()),
+					)?;
+					Some(route)
+				}
+			}
+		};
+
 		let mut storage = self.storage.write();
-		storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification));
-		storage.hashes.insert(number, hash.clone());
+
+		storage.leaves.import(hash.clone(), number.clone(), header.parent_hash().clone());
 
 		if new_state.is_best() {
+			if let Some(tree_route) = best_tree_route {
+				// apply retraction and enaction when reorganizing up to parent hash
+				let enacted = tree_route.enacted();
+
+				for entry in enacted {
+					storage.hashes.insert(entry.number, entry.hash);
+				}
+
+				for entry in tree_route.retracted().iter().skip(enacted.len()) {
+					storage.hashes.remove(&entry.number);
+				}
+			}
+
 			storage.best_hash = hash.clone();
 			storage.best_number = number.clone();
+			storage.hashes.insert(number, hash.clone());
 		}
+
+		storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification));
+
 		if let NewBlockState::Final = new_state {
 			storage.finalized_hash = hash;
 		}
@@ -174,6 +211,8 @@ impl<Block: BlockT> Blockchain<Block> {
 		if number == Zero::zero() {
 			storage.genesis_hash = hash;
 		}
+
+		Ok(())
 	}
 
 	/// Compare this blockchain with another in-mem blockchain
@@ -262,6 +301,10 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
 	fn cache(&self) -> Option<&blockchain::Cache<Block>> {
 		Some(&self.cache)
 	}
+
+	fn leaves(&self) -> error::Result<Vec<Block::Hash>> {
+		Ok(self.storage.read().leaves.hashes())
+	}
 }
 
 impl<Block: BlockT> light::blockchain::Storage<Block> for Blockchain<Block>
@@ -276,7 +319,7 @@ impl<Block: BlockT> light::blockchain::Storage<Block> for Blockchain<Block>
 	) -> error::Result<()> {
 		let hash = header.hash();
 		let parent_hash = *header.parent_hash();
-		self.insert(hash, header, None, None, state);
+		self.insert(hash, header, None, None, state)?;
 		if state.is_best() {
 			self.cache.insert(parent_hash, authorities);
 		}
@@ -436,7 +479,7 @@ where
 				}
 			}
 
-			self.blockchain.insert(hash, header, justification, body, pending_block.state);
+			self.blockchain.insert(hash, header, justification, body, pending_block.state)?;
 			// dumb implementation - store value for each block
 			if pending_block.state.is_best() {
 				self.blockchain.cache.insert(parent_hash, operation.pending_authorities);
@@ -501,3 +544,26 @@ pub fn cache_authorities_at<Block: BlockT>(
 ) {
 	blockchain.cache.insert(at, authorities);
 }
+
+#[cfg(test)]
+mod tests {
+	use std::sync::Arc;
+	use test_client;
+	use primitives::Blake2Hasher;
+
+	type TestBackend = test_client::client::in_mem::Backend<test_client::runtime::Block, Blake2Hasher>;
+
+	#[test]
+	fn test_leaves_with_complex_block_tree() {
+		let backend = Arc::new(TestBackend::new());
+
+		test_client::trait_tests::test_leaves_for_backend(backend);
+	}
+
+	#[test]
+	fn test_blockchain_query_by_number_gets_canonical() {
+		let backend = Arc::new(TestBackend::new());
+
+		test_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend);
+	}
+}