best_containing operations (issue 603) (#740)

* add stub for Client.best_chain_containing_block_hash * add fn blockchain::Backend::leaf_hashes * fix typo * sketch out Client.best_chain_containing_block_hash * fix indent * Blockchain.leaf_hashes -> Blockchain.leaves * add unimplemented! stub impls for Blockchain.leaves * start impl of Blockchain.leaves for in-memory client db * Client.best_chain_containing...: check canonical first and make compile * first rough attempt at maintaining leaf list in in-memory db * fix tab indent * add test best_chain_containing_single_block * add failing test best_chain_containing_with_fork * pub use client::blockchain; in test-client to prevent circular dep in client tests * best_chain_containing_with_single_block: improve and test leaves * far improve in-memory Backend::leaves impl * test blockchain::Backend::leaves more thoroughly * handle more edge cases in blockchain::Backend::leaves impl for in memory * fix test best_chain_containing_with_fork (two distinct test blocks had same hash) * make best_chain_containing_block_hash pass existing tests * improve docstring for Blockchain::leaves * Client.best_chain_containing: some cleanup. support max_block_number * best_chain_containing: remove broken outcommented fast check for best = canonical * remove blank line * best_block_containing: return None if target_hash not found * best_chain_containing: add unreachable! at end of function * improve tests for best_chain_containing * renames * more elaborate test scenario for best_containing * best_containing: fix restriction of search through maybe_max_number * best_containing: tests for restriction of search * get rid of unnecessary clones * replace Client::new_in_mem by new_with_backend which is useful for testing backends * add test_client::new_with_backend for testing different backend impls * add test for in_mem::Backend::leaves * remove unused imports * in_mem test_leaves: simplify * flesh out tests for in_mem leaves impl * remove tests for leaves from client which are now covered in implementing module * improve comment * add Client.new_in_mem again * unwrap in test_client::new_with_backend * make test_client::BlockBuilderExt work not just with in-mem backend * make test client ext not just work with in mem backend * add failing Backend.leaves test for client-db * update Cargo.lock * replace KeccakHasher with Blake2Hasher * refactor address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217822862 * refactor using NumberFor address grumble https://github.com/paritytech/substrate/pull/740#discussion_r217823341 * add test that exposes possible problem * update docstring for Client.best_containing * extract test for Backend.leaves for reuse * improve test blockchain_header_and_hash_return_blocks_from_canonical_chain_given_block_numbers * extract test_blockchain_query_by_number_gets_canonical to easily test multiple impls * remove whitespace * remove todo * Client.best_containing: pre-empt search loop when target in canonical * best_containing: prevent race condition by holding import lock * add todo * extract leaf list update code into function * add comment * client-db: use in-memory-kvdb for tests * use BTreeSet to store leaves for in-mem which is faster and simpler * add docstring * add comments and fix formatting * add initial raw version of LeafSet * remove Client::update_leaves which has been superceded by LeafSet * use LeafSet in in-mem backend * keccak -> blake2 * don't reexport codec traits in primitives addresses https://github.com/paritytech/substrate/pull/740#discussion_r219538185 * fix rebase mistake * improve LeafSet and use it in state-db * correct Transfer nonces to fix ApplyExtinsicFailed(Stale) * use given backend in canoncal test * kill dead tree-route code in util * fix warnings * tests for leafset * reorganizations in in_mem backend * fix reorganization canon block logic * DB commit and safe reversion on write error * fix style nits
2026-06-12 13:31:10 +00:00 · 2018-09-26 20:34:05 +09:00
parent 1438e15925
commit 58cc0992df
16 changed files with 1041 additions and 57 deletions
@@ -0,0 +1,204 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::collections::BTreeSet;
+use std::cmp::{Ord, Ordering};
+use kvdb::{KeyValueDB, DBTransaction};
+use runtime_primitives::traits::SimpleArithmetic;
+use codec::{Encode, Decode};
+use error;
+
+/// helper wrapper type to keep a list of block hashes ordered
+/// by `number` descending in a `BTreeSet` which allows faster and simpler
+/// insertion and removal than keeping them in a list.
+#[derive(Debug, Clone)]
+struct LeafSetItem<H, N> {
+	hash: H,
+	number: N,
+}
+
+impl<H, N> Ord for LeafSetItem<H, N> where N: Ord {
+	fn cmp(&self, other: &Self) -> Ordering {
+		// reverse (descending) order
+		other.number.cmp(&self.number)
+	}
+}
+
+impl<H, N> PartialOrd for LeafSetItem<H, N> where N: PartialOrd {
+	fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+		// reverse (descending) order
+		other.number.partial_cmp(&self.number)
+	}
+}
+
+impl<H, N> PartialEq for LeafSetItem<H, N> where N: PartialEq {
+	fn eq(&self, other: &LeafSetItem<H, N>) -> bool {
+		self.number == other.number
+	}
+}
+
+impl<H, N> Eq for LeafSetItem<H, N> where N: PartialEq {}
+
+/// A displaced leaf after import.
+pub struct DisplacedLeaf<H, N> {
+	new_hash: H,
+	displaced: LeafSetItem<H, N>,
+}
+
+/// list of leaf hashes ordered by number (descending).
+/// stored in memory for fast access.
+/// this allows very fast checking and modification of active leaves.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LeafSet<H, N> {
+	storage: BTreeSet<LeafSetItem<H, N>>,
+}
+
+impl<H, N> LeafSet<H, N> where
+	H: Clone + Decode + Encode,
+	N: Clone + SimpleArithmetic + Decode + Encode,
+{
+	/// Construct a new, blank leaf set.
+	pub fn new() -> Self {
+		Self {
+			storage: BTreeSet::new()
+		}
+	}
+
+	/// Read the leaf list from the DB, using given prefix for keys.
+	pub fn read_from_db(db: &KeyValueDB, column: Option<u32>, prefix: &[u8]) -> error::Result<Self> {
+		let mut storage = BTreeSet::new();
+
+		for (key, value) in db.iter_from_prefix(column, prefix) {
+			let raw_hash = &mut &key[prefix.len()..];
+			let hash = match Decode::decode(raw_hash) {
+				Some(hash) => hash,
+				None => return Err(error::ErrorKind::Backend("Error decoding hash".into()).into()),
+			};
+			let number = match Decode::decode(&mut &value[..]) {
+				Some(number) => number,
+				None => return Err(error::ErrorKind::Backend("Error decoding number".into()).into()),
+			};
+			storage.insert(LeafSetItem { hash, number });
+		}
+		Ok(Self { storage })
+	}
+
+	/// update the leaf list on import. returns a displaced leaf if there was one.
+	pub fn import(&mut self, hash: H, number: N, parent_hash: H) -> Option<DisplacedLeaf<H, N>> {
+		// avoid underflow for genesis.
+		let displaced = if number != N::zero() {
+			let displaced = LeafSetItem {
+				hash: parent_hash,
+				number: number.clone() - N::one(),
+			};
+			let was_displaced = self.storage.remove(&displaced);
+
+			if was_displaced {
+				Some(DisplacedLeaf {
+					new_hash: hash.clone(),
+					displaced,
+				})
+			} else {
+				None
+			}
+		} else {
+			None
+		};
+
+		self.storage.insert(LeafSetItem { hash, number });
+		displaced
+	}
+
+	/// Undo an import operation, with a displaced leaf.
+	pub fn undo(&mut self, displaced: DisplacedLeaf<H, N>) {
+		let new_number = displaced.displaced.number.clone() + N::one();
+		self.storage.remove(&LeafSetItem { hash: displaced.new_hash, number: new_number });
+		self.storage.insert(displaced.displaced);
+	}
+
+	/// currently since revert only affects the canonical chain
+	/// we assume that parent has no further children
+	/// and we add it as leaf again
+	pub fn revert(&mut self, hash: H, number: N, parent_hash: H) {
+		self.storage.insert(LeafSetItem {
+			hash: parent_hash,
+			number: number.clone() - N::one(),
+		});
+		self.storage.remove(&LeafSetItem { hash, number });
+	}
+
+	/// returns an iterator over all hashes in the leaf set
+	/// ordered by their block number descending.
+	pub fn hashes(&self) -> Vec<H> {
+		self.storage.iter().map(|item| item.hash.clone()).collect()
+	}
+
+	/// Write the leaf list to the database transaction.
+	pub fn prepare_transaction(&self, tx: &mut DBTransaction, column: Option<u32>, prefix: &[u8]) {
+		let mut buf = prefix.to_vec();
+		for &LeafSetItem { ref hash, ref number } in &self.storage {
+			hash.using_encoded(|s| buf.extend(s));
+			tx.put_vec(column, &buf[..], number.encode());
+			buf.truncate(prefix.len()); // reuse allocation.
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn it_works() {
+		let mut set = LeafSet::new();
+		set.import(0u32, 0u32, 0u32);
+
+		set.import(1_1, 1, 0);
+		set.import(2_1, 2, 1_1);
+		set.import(3_1, 3, 2_1);
+
+		assert!(set.storage.contains(&LeafSetItem { hash: 3_1, number: 3 }));
+		assert!(!set.storage.contains(&LeafSetItem { hash: 2_1, number: 2 }));
+		assert!(!set.storage.contains(&LeafSetItem { hash: 1_1, number: 1 }));
+		assert!(!set.storage.contains(&LeafSetItem { hash: 0, number: 0 }));
+
+		set.import(2_2, 2, 1_1);
+
+		assert!(set.storage.contains(&LeafSetItem { hash: 3_1, number: 3 }));
+		assert!(set.storage.contains(&LeafSetItem { hash: 2_2, number: 2 }));
+	}
+
+	#[test]
+	fn flush_to_disk() {
+		const PREFIX: &[u8] = b"abcdefg";
+		let db = ::kvdb_memorydb::create(0);
+
+		let mut set = LeafSet::new();
+		set.import(0u32, 0u32, 0u32);
+
+		set.import(1_1, 1, 0);
+		set.import(2_1, 2, 1_1);
+		set.import(3_1, 3, 2_1);
+
+		let mut tx = DBTransaction::new();
+
+		set.prepare_transaction(&mut tx, None, PREFIX);
+		db.write(tx).unwrap();
+
+		let set2 = LeafSet::read_from_db(&db, None, PREFIX).unwrap();
+		assert_eq!(set, set2);
+	}
+}