Phase 1 of repo reorg (#719)

* Remove unneeded script * Rename Substrate Demo -> Substrate * Rename demo -> node * Build wasm from last rename. * Merge ed25519 into substrate-primitives * Minor tweak * Rename substrate -> core * Move substrate-runtime-support to core/runtime/support * Rename/move substrate-runtime-version * Move codec up a level * Rename substrate-codec -> parity-codec * Move environmental up a level * Move pwasm-* up to top, ready for removal * Remove requirement of s-r-support from s-r-primitives * Move core/runtime/primitives into core/runtime-primitives * Remove s-r-support dep from s-r-version * Remove dep of s-r-support from bft * Remove dep of s-r-support from node/consensus * Sever all other core deps from s-r-support * Forgot the no_std directive * Rename non-SRML modules to sr-* to avoid match clashes * Move runtime/* to srml/* * Rename substrate-runtime-* -> srml-* * Move srml to top-level
2026-06-14 02:51:08 +00:00 · 2018-09-12 11:13:31 +02:00
parent 8fe5aa4c81
commit 1e01162505
374 changed files with 2845 additions and 2902 deletions
@@ -0,0 +1,407 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! State database maintenance. Handles finalization and pruning in the database. The input to
+//! this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that
+//! were added or deleted during block execution.
+//!
+//! # Finalization.
+//! Finalization window tracks a tree of blocks identified by header hash. The in-memory
+//! overlay allows to get any node that was was inserted in any any of the blocks within the window.
+//! The tree is journaled to the backing database and rebuilt on startup.
+//! Finalization function select one root from the top of the tree and discards all other roots and
+//! their subtrees.
+//!
+//! # Pruning.
+//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each finalization until pruning
+//! constraints are satisfied.
+//!
+// end::description[]
+
+#[macro_use] extern crate log;
+#[macro_use] extern crate parity_codec_derive;
+extern crate parking_lot;
+extern crate parity_codec as codec;
+extern crate substrate_primitives as primitives;
+
+mod unfinalized;
+mod pruning;
+#[cfg(test)] mod test;
+
+use std::fmt;
+use parking_lot::RwLock;
+use codec::Codec;
+use std::collections::HashSet;
+use unfinalized::UnfinalizedOverlay;
+use pruning::RefWindow;
+
+/// Database value type.
+pub type DBValue = Vec<u8>;
+
+/// Basic set of requirements for the Block hash and node key types.
+pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {}
+impl<T: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static> Hash for T {}
+
+/// Backend database trait. Read-only.
+pub trait MetaDb {
+	type Error: fmt::Debug;
+
+	/// Get meta value, such as the journal.
+	fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, Self::Error>;
+}
+
+
+/// Backend database trait. Read-only.
+pub trait HashDb {
+	type Hash: Hash;
+	type Error: fmt::Debug;
+
+	/// Get state trie node.
+	fn get(&self, key: &Self::Hash) -> Result<Option<DBValue>, Self::Error>;
+}
+
+/// Error type.
+pub enum Error<E: fmt::Debug> {
+	/// Database backend error.
+	Db(E),
+	/// `Codec` decoding error.
+	Decoding,
+}
+
+impl<E: fmt::Debug> fmt::Debug for Error<E> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+		match self {
+			Error::Db(e) => e.fmt(f),
+			Error::Decoding => write!(f, "Error decoding slicable value"),
+		}
+	}
+}
+
+/// A set of state node changes.
+#[derive(Default, Debug, Clone)]
+pub struct ChangeSet<H: Hash> {
+	/// Inserted nodes.
+	pub inserted: Vec<(H, DBValue)>,
+	/// Deleted nodes.
+	pub deleted: Vec<H>,
+}
+
+
+/// A set of changes to the backing database.
+#[derive(Default, Debug, Clone)]
+pub struct CommitSet<H: Hash> {
+	/// State node changes.
+	pub data: ChangeSet<H>,
+	/// Metadata changes.
+	pub meta: ChangeSet<Vec<u8>>,
+}
+
+/// Pruning constraints. If none are specified pruning is
+#[derive(Default, Debug, Clone)]
+pub struct Constraints {
+	/// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only unfinalized states.
+	pub max_blocks: Option<u32>,
+	/// Maximum memory in the pruning overlay.
+	pub max_mem: Option<usize>,
+}
+
+/// Pruning mode.
+#[derive(Debug, Clone)]
+pub enum PruningMode {
+	/// Maintain a pruning window.
+	Constrained(Constraints),
+	/// No pruning. Finalization is a no-op.
+	ArchiveAll,
+	/// Finalization discards unfinalized nodes. All the finalized nodes are kept in the DB.
+	ArchiveCanonical,
+}
+
+impl PruningMode {
+	/// Create a mode that keeps given number of blocks.
+	pub fn keep_blocks(n: u32) -> PruningMode {
+		PruningMode::Constrained(Constraints {
+			max_blocks: Some(n),
+			max_mem: None,
+		})
+	}
+}
+
+impl Default for PruningMode {
+	fn default() -> Self {
+		PruningMode::keep_blocks(256)
+	}
+}
+
+fn to_meta_key<S: Codec>(suffix: &[u8], data: &S) -> Vec<u8> {
+	let mut buffer = data.encode();
+	buffer.extend(suffix);
+	buffer
+}
+
+struct StateDbSync<BlockHash: Hash, Key: Hash> {
+	mode: PruningMode,
+	unfinalized: UnfinalizedOverlay<BlockHash, Key>,
+	pruning: Option<RefWindow<BlockHash, Key>>,
+	pinned: HashSet<BlockHash>,
+}
+
+impl<BlockHash: Hash, Key: Hash> StateDbSync<BlockHash, Key> {
+	pub fn new<D: MetaDb>(mode: PruningMode, db: &D) -> Result<StateDbSync<BlockHash, Key>, Error<D::Error>> {
+		trace!("StateDb settings: {:?}", mode);
+		let unfinalized: UnfinalizedOverlay<BlockHash, Key> = UnfinalizedOverlay::new(db)?;
+		let pruning: Option<RefWindow<BlockHash, Key>> = match mode {
+			PruningMode::Constrained(Constraints {
+				max_mem: Some(_),
+				..
+			}) => unimplemented!(),
+			PruningMode::Constrained(_) => Some(RefWindow::new(db)?),
+			PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None,
+		};
+		Ok(StateDbSync {
+			mode,
+			unfinalized,
+			pruning: pruning,
+			pinned: Default::default(),
+		})
+	}
+
+	pub fn insert_block(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet<Key>) -> CommitSet<Key> {
+		if number == 0 {
+			return CommitSet {
+				data: changeset,
+				meta: Default::default(),
+			}
+		}
+		match self.mode {
+			PruningMode::ArchiveAll => {
+				changeset.deleted.clear();
+				// write changes immediately
+				CommitSet {
+					data: changeset,
+					meta: Default::default(),
+				}
+			},
+			PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => {
+				self.unfinalized.insert(hash, number, parent_hash, changeset)
+			}
+		}
+	}
+
+	pub fn finalize_block(&mut self, hash: &BlockHash) -> CommitSet<Key> {
+		// clear the temporary overlay from the previous finalization.
+		self.unfinalized.clear_overlay();
+		let mut commit = match self.mode {
+			PruningMode::ArchiveAll => {
+				CommitSet::default()
+			},
+			PruningMode::ArchiveCanonical => {
+				let mut commit = self.unfinalized.finalize(hash);
+				commit.data.deleted.clear();
+				commit
+			},
+			PruningMode::Constrained(_) => {
+				self.unfinalized.finalize(hash)
+			},
+		};
+		if let Some(ref mut pruning) = self.pruning {
+			pruning.note_finalized(hash, &mut commit);
+		}
+		self.prune(&mut commit);
+		commit
+	}
+
+	pub fn best_finalized(&self) -> u64 {
+		return self.unfinalized.last_finalized_block_number()
+	}
+
+	pub fn is_pruned(&self, number: u64) -> bool {
+		self.pruning.as_ref().map_or(false, |pruning| number < pruning.pending())
+	}
+
+	fn prune(&mut self, commit: &mut CommitSet<Key>) {
+		if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) = (&mut self.pruning, &self.mode) {
+			loop {
+				if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 {
+					break;
+				}
+
+				if constraints.max_mem.map_or(false, |m| pruning.mem_used() > m) {
+					break;
+				}
+
+				let pinned = &self.pinned;
+				if pruning.next_hash().map_or(false, |h| pinned.contains(&h)) {
+					break;
+				}
+				pruning.prune_one(commit);
+			}
+		}
+	}
+
+	/// Revert all unfinalized blocks with the best block number.
+	/// Returns a database commit or `None` if not possible.
+	/// For archive an empty commit set is returned.
+	pub fn revert_one(&mut self) -> Option<CommitSet<Key>> {
+		match self.mode {
+			PruningMode::ArchiveAll => {
+				Some(CommitSet::default())
+			},
+			PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
+				self.unfinalized.revert_one()
+			},
+		}
+	}
+
+	pub fn pin(&mut self, hash: &BlockHash) {
+		self.pinned.insert(hash.clone());
+	}
+
+	pub fn unpin(&mut self, hash: &BlockHash) {
+		self.pinned.remove(hash);
+	}
+
+	pub fn get<D: HashDb<Hash=Key>>(&self, key: &Key, db: &D) -> Result<Option<DBValue>, Error<D::Error>> {
+		if let Some(value) = self.unfinalized.get(key) {
+			return Ok(Some(value));
+		}
+		db.get(key).map_err(|e| Error::Db(e))
+	}
+}
+
+/// State DB maintenance. See module description.
+/// Can be shared across threads.
+pub struct StateDb<BlockHash: Hash, Key: Hash> {
+	db: RwLock<StateDbSync<BlockHash, Key>>,
+}
+
+impl<BlockHash: Hash, Key: Hash> StateDb<BlockHash, Key> {
+	/// Creates a new instance. Does not expect any metadata in the database.
+	pub fn new<D: MetaDb>(mode: PruningMode, db: &D) -> Result<StateDb<BlockHash, Key>, Error<D::Error>> {
+		Ok(StateDb {
+			db: RwLock::new(StateDbSync::new(mode, db)?)
+		})
+	}
+
+	/// Add a new unfinalized block.
+	pub fn insert_block(&self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet<Key>) -> CommitSet<Key> {
+		self.db.write().insert_block(hash, number, parent_hash, changeset)
+	}
+
+	/// Finalize a previously inserted block.
+	pub fn finalize_block(&self, hash: &BlockHash) -> CommitSet<Key> {
+		self.db.write().finalize_block(hash)
+	}
+
+	/// Prevents pruning of specified block and its descendants.
+	pub fn pin(&self, hash: &BlockHash) {
+		self.db.write().pin(hash)
+	}
+
+	/// Allows pruning of specified block.
+	pub fn unpin(&self, hash: &BlockHash) {
+		self.db.write().unpin(hash)
+	}
+
+	/// Get a value from unfinalized/pruning overlay or the backing DB.
+	pub fn get<D: HashDb<Hash=Key>>(&self, key: &Key, db: &D) -> Result<Option<DBValue>, Error<D::Error>> {
+		self.db.read().get(key, db)
+	}
+
+	/// Revert all unfinalized blocks with the best block number.
+	/// Returns a database commit or `None` if not possible.
+	/// For archive an empty commit set is returned.
+	pub fn revert_one(&self) -> Option<CommitSet<Key>> {
+		self.db.write().revert_one()
+	}
+
+	/// Returns last finalized block number.
+	pub fn best_finalized(&self) -> u64 {
+		return self.db.read().best_finalized()
+	}
+
+	/// Check if block is pruned away.
+	pub fn is_pruned(&self, number: u64) -> bool {
+		return self.db.read().is_pruned(number)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::H256;
+	use {StateDb, PruningMode, Constraints};
+	use test::{make_db, make_changeset, TestDb};
+
+	fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256>) {
+		let mut db = make_db(&[91, 921, 922, 93, 94]);
+		let state_db = StateDb::new(settings, &db).unwrap();
+
+		db.commit(&state_db.insert_block(&H256::from(1), 1, &H256::from(0), make_changeset(&[1], &[91])));
+		db.commit(&state_db.insert_block(&H256::from(21), 2, &H256::from(1), make_changeset(&[21], &[921, 1])));
+		db.commit(&state_db.insert_block(&H256::from(22), 2, &H256::from(1), make_changeset(&[22], &[922])));
+		db.commit(&state_db.insert_block(&H256::from(3), 3, &H256::from(21), make_changeset(&[3], &[93])));
+		db.commit(&state_db.finalize_block(&H256::from(1)));
+		db.commit(&state_db.insert_block(&H256::from(4), 4, &H256::from(3), make_changeset(&[4], &[94])));
+		db.commit(&state_db.finalize_block(&H256::from(21)));
+		db.commit(&state_db.finalize_block(&H256::from(3)));
+
+		(db, state_db)
+	}
+
+	#[test]
+	fn full_archive_keeps_everything() {
+		let (db, _) = make_test_db(PruningMode::ArchiveAll);
+		assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94])));
+	}
+
+	#[test]
+	fn canonical_archive_keeps_canonical() {
+		let (db, _) = make_test_db(PruningMode::ArchiveCanonical);
+		assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94])));
+	}
+
+	#[test]
+	fn prune_window_0() {
+		let (db, _) = make_test_db(PruningMode::Constrained(Constraints {
+			max_blocks: Some(0),
+			max_mem: None,
+		}));
+		assert!(db.data_eq(&make_db(&[21, 3, 922, 94])));
+	}
+
+	#[test]
+	fn prune_window_1() {
+		let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints {
+			max_blocks: Some(1),
+			max_mem: None,
+		}));
+		assert!(sdb.is_pruned(0));
+		assert!(sdb.is_pruned(1));
+		assert!(!sdb.is_pruned(2));
+		assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94])));
+	}
+
+	#[test]
+	fn prune_window_2() {
+		let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints {
+			max_blocks: Some(2),
+			max_mem: None,
+		}));
+		assert!(sdb.is_pruned(0));
+		assert!(!sdb.is_pruned(1));
+		assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94])));
+	}
+}
@@ -0,0 +1,267 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Pruning window.
+//!
+//! For each block we maintain a list of nodes pending deletion.
+//! There is also a global index of node key to block number.
+//! If a node is re-inserted into the window it gets removed from
+//! the death list.
+//! The changes are journaled in the DB.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use codec::{Encode, Decode};
+use {CommitSet, Error, MetaDb, to_meta_key, Hash};
+
+const LAST_PRUNED: &[u8] = b"last_pruned";
+const PRUNING_JOURNAL: &[u8] = b"pruning_journal";
+
+/// See module documentation.
+pub struct RefWindow<BlockHash: Hash, Key: Hash> {
+	death_rows: VecDeque<DeathRow<BlockHash, Key>>,
+	death_index: HashMap<Key, u64>,
+	pending_number: u64,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct DeathRow<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	journal_key: Vec<u8>,
+	deleted: HashSet<Key>,
+}
+
+#[derive(Encode, Decode)]
+struct JournalRecord<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	inserted: Vec<Key>,
+	deleted: Vec<Key>,
+}
+
+fn to_journal_key(block: u64) -> Vec<u8> {
+	to_meta_key(PRUNING_JOURNAL, &block)
+}
+
+impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
+	pub fn new<D: MetaDb>(db: &D) -> Result<RefWindow<BlockHash, Key>, Error<D::Error>> {
+		let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &()))
+			.map_err(|e| Error::Db(e))?;
+		let pending_number: u64 = match last_pruned {
+			Some(buffer) => u64::decode(&mut buffer.as_slice()).ok_or(Error::Decoding)? + 1,
+			None => 0,
+		};
+		let mut block = pending_number;
+		let mut pruning = RefWindow {
+			death_rows: Default::default(),
+			death_index: Default::default(),
+			pending_number: pending_number,
+		};
+		// read the journal
+		trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number);
+		loop {
+			let journal_key = to_journal_key(block);
+			match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
+				Some(record) => {
+					let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice()).ok_or(Error::Decoding)?;
+					trace!(target: "state-db", "Pruning journal entry {} ({} inserted, {} deleted)", block, record.inserted.len(), record.deleted.len());
+					pruning.import(&record.hash, journal_key, record.inserted.into_iter(), record.deleted);
+				},
+				None => break,
+			}
+			block += 1;
+		}
+		Ok(pruning)
+	}
+
+	fn import<I: IntoIterator<Item=Key>>(&mut self, hash: &BlockHash, journal_key: Vec<u8>, inserted: I, deleted: Vec<Key>) {
+		// remove all re-inserted keys from death rows
+		for k in inserted {
+			if let Some(block) = self.death_index.remove(&k) {
+				self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k);
+			}
+		}
+
+		// add new keys
+		let imported_block = self.pending_number + self.death_rows.len() as u64;
+		for k in deleted.iter() {
+			self.death_index.insert(k.clone(), imported_block);
+		}
+		self.death_rows.push_back(
+			DeathRow {
+				hash: hash.clone(),
+				deleted: deleted.into_iter().collect(),
+				journal_key: journal_key,
+			}
+		);
+	}
+
+	pub fn window_size(&self) -> u64 {
+		self.death_rows.len() as u64
+	}
+
+	pub fn next_hash(&self) -> Option<BlockHash> {
+		self.death_rows.front().map(|r| r.hash.clone())
+	}
+
+	pub fn mem_used(&self) -> usize {
+		0
+	}
+
+	pub fn pending(&self) -> u64 {
+		self.pending_number
+	}
+
+	/// Prune next block. Expects at least one block in the window. Adds changes to `commit`.
+	pub fn prune_one(&mut self, commit: &mut CommitSet<Key>) {
+		let pruned = self.death_rows.pop_front().expect("prune_one is only called with a non-empty window");
+		trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len());
+		for k in pruned.deleted.iter() {
+			self.death_index.remove(&k);
+		}
+		commit.data.deleted.extend(pruned.deleted.into_iter());
+		commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), self.pending_number.encode()));
+		commit.meta.deleted.push(pruned.journal_key);
+		self.pending_number += 1;
+	}
+
+	/// Add a change set to the window. Creates a journal record and pushes it to `commit`
+	pub fn note_finalized(&mut self, hash: &BlockHash, commit: &mut CommitSet<Key>) {
+		trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len());
+		let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect();
+		let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new());
+		let journal_record = JournalRecord {
+			hash: hash.clone(),
+			inserted,
+			deleted,
+		};
+		let block = self.pending_number + self.window_size();
+		let journal_key = to_journal_key(block);
+		commit.meta.inserted.push((journal_key.clone(), journal_record.encode()));
+
+		self.import(hash, journal_key, journal_record.inserted.into_iter(), journal_record.deleted);
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::RefWindow;
+	use primitives::H256;
+	use {CommitSet};
+	use test::{make_db, make_commit, TestDb};
+
+	fn check_journal(pruning: &RefWindow<H256, H256>, db: &TestDb) {
+		let restored: RefWindow<H256, H256> = RefWindow::new(db).unwrap();
+		assert_eq!(pruning.pending_number, restored.pending_number);
+		assert_eq!(pruning.death_rows, restored.death_rows);
+		assert_eq!(pruning.death_index, restored.death_index);
+	}
+
+	#[test]
+	fn created_from_empty_db() {
+		let db = make_db(&[]);
+		let pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
+		assert_eq!(pruning.pending_number, 0);
+		assert!(pruning.death_rows.is_empty());
+		assert!(pruning.death_index.is_empty());
+	}
+
+	#[test]
+	#[should_panic]
+	fn prune_empty_panics() {
+		let db = make_db(&[]);
+		let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+	}
+
+	#[test]
+	fn prune_one() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
+		let mut commit = make_commit(&[4, 5], &[1, 3]);
+		let h = H256::random();
+		pruning.note_finalized(&h, &mut commit);
+		db.commit(&commit);
+		assert!(commit.data.deleted.is_empty());
+		assert_eq!(pruning.death_rows.len(), 1);
+		assert_eq!(pruning.death_index.len(), 2);
+		assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[2, 4, 5])));
+		assert!(pruning.death_rows.is_empty());
+		assert!(pruning.death_index.is_empty());
+		assert_eq!(pruning.pending_number, 1);
+	}
+
+	#[test]
+	fn prune_two() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
+		let mut commit = make_commit(&[4], &[1]);
+		pruning.note_finalized(&H256::random(), &mut commit);
+		db.commit(&commit);
+		let mut commit = make_commit(&[5], &[2]);
+		pruning.note_finalized(&H256::random(), &mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
+
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[2, 3, 4, 5])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[3, 4, 5])));
+		assert_eq!(pruning.pending_number, 2);
+	}
+
+	#[test]
+	fn reinserted_survives() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_finalized(&H256::random(), &mut commit);
+		db.commit(&commit);
+		let mut commit = make_commit(&[2], &[]);
+		pruning.note_finalized(&H256::random(), &mut commit);
+		db.commit(&commit);
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_finalized(&H256::random(), &mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		pruning.prune_one(&mut commit);
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 3])));
+		assert_eq!(pruning.pending_number, 3);
+	}
+}
@@ -0,0 +1,83 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test utils
+
+use std::collections::HashMap;
+use primitives::H256;
+use {DBValue, ChangeSet, CommitSet, MetaDb, HashDb};
+
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
+pub struct TestDb {
+	pub data: HashMap<H256, DBValue>,
+	pub meta: HashMap<Vec<u8>, DBValue>,
+}
+
+impl MetaDb for TestDb {
+	type Error = ();
+
+	fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, ()> {
+		Ok(self.meta.get(key).cloned())
+	}
+}
+
+impl HashDb for TestDb {
+	type Error = ();
+	type Hash = H256;
+
+	fn get(&self, key: &H256) -> Result<Option<DBValue>, ()> {
+		Ok(self.data.get(key).cloned())
+	}
+}
+
+impl TestDb {
+	pub fn commit(&mut self, commit: &CommitSet<H256>) {
+		self.data.extend(commit.data.inserted.iter().cloned());
+		for k in commit.data.deleted.iter() {
+			self.data.remove(k);
+		}
+		self.meta.extend(commit.meta.inserted.iter().cloned());
+		for k in commit.meta.deleted.iter() {
+			self.meta.remove(k);
+		}
+	}
+
+	pub fn data_eq(&self, other: &TestDb) -> bool {
+		self.data == other.data
+	}
+}
+
+pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet<H256> {
+	ChangeSet {
+		inserted: inserted.iter().map(|v| (H256::from(*v), H256::from(*v).to_vec())).collect(),
+		deleted: deleted.iter().map(|v| H256::from(*v)).collect(),
+	}
+}
+
+pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet<H256> {
+	CommitSet {
+		data: make_changeset(inserted, deleted),
+		meta: ChangeSet::default(),
+	}
+}
+
+pub fn make_db(inserted: &[u64]) -> TestDb {
+	TestDb {
+		data: inserted.iter().map(|v| (H256::from(*v), H256::from(*v).to_vec())).collect(),
+		meta: Default::default(),
+	}
+}
+
@@ -0,0 +1,533 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Finalization window.
+//! Maintains trees of block overlays and allows discarding trees/roots
+//! The overlays are added in `insert` and removed in `finalize`.
+//! Last finalized overlay is kept in memory until next call to `finalize` or
+//! `clear_overlay`
+
+use std::collections::{HashMap, VecDeque};
+use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key};
+use codec::{Decode, Encode};
+
+const UNFINALIZED_JOURNAL: &[u8] = b"unfinalized_journal";
+const LAST_FINALIZED: &[u8] = b"last_finalized";
+
+/// See module documentation.
+pub struct UnfinalizedOverlay<BlockHash: Hash, Key: Hash> {
+	last_finalized: Option<(BlockHash, u64)>,
+	levels: VecDeque<Vec<BlockOverlay<BlockHash, Key>>>,
+	parents: HashMap<BlockHash, BlockHash>,
+	last_finalized_overlay: HashMap<Key, DBValue>,
+}
+
+#[derive(Encode, Decode)]
+struct JournalRecord<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	parent_hash: BlockHash,
+	inserted: Vec<(Key, DBValue)>,
+	deleted: Vec<Key>,
+}
+
+fn to_journal_key(block: u64, index: u64) -> Vec<u8> {
+	to_meta_key(UNFINALIZED_JOURNAL, &(block, index))
+}
+
+#[cfg_attr(test, derive(PartialEq, Debug))]
+struct BlockOverlay<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	journal_key: Vec<u8>,
+	values: HashMap<Key, DBValue>,
+	deleted: Vec<Key>,
+}
+
+impl<BlockHash: Hash, Key: Hash> UnfinalizedOverlay<BlockHash, Key> {
+	/// Creates a new instance. Does not expect any metadata to be present in the DB.
+	pub fn new<D: MetaDb>(db: &D) -> Result<UnfinalizedOverlay<BlockHash, Key>, Error<D::Error>> {
+		let last_finalized = db.get_meta(&to_meta_key(LAST_FINALIZED, &()))
+			.map_err(|e| Error::Db(e))?;
+		let last_finalized = match last_finalized {
+			Some(buffer) => Some(<(BlockHash, u64)>::decode(&mut buffer.as_slice()).ok_or(Error::Decoding)?),
+			None => None,
+		};
+		let mut levels = VecDeque::new();
+		let mut parents = HashMap::new();
+		if let Some((ref hash, mut block)) = last_finalized {
+			// read the journal
+			trace!(target: "state-db", "Reading unfinalized journal. Last finalized #{} ({:?})", block, hash);
+			let mut total: u64 = 0;
+			block += 1;
+			loop {
+				let mut index: u64 = 0;
+				let mut level = Vec::new();
+				loop {
+					let journal_key = to_journal_key(block, index);
+					match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
+						Some(record) => {
+							let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice()).ok_or(Error::Decoding)?;
+							let overlay = BlockOverlay {
+								hash: record.hash.clone(),
+								journal_key,
+								values: record.inserted.into_iter().collect(),
+								deleted: record.deleted,
+							};
+							trace!(target: "state-db", "Unfinalized journal entry {}.{} ({} inserted, {} deleted)", block, index, overlay.values.len(), overlay.deleted.len());
+							level.push(overlay);
+							parents.insert(record.hash, record.parent_hash);
+							index += 1;
+							total += 1;
+						},
+						None => break,
+					}
+				}
+				if level.is_empty() {
+					break;
+				}
+				levels.push_back(level);
+				block += 1;
+			}
+			trace!(target: "state-db", "Finished reading unfinalized journal, {} entries", total);
+		}
+		Ok(UnfinalizedOverlay {
+			last_finalized: last_finalized,
+			levels,
+			parents,
+			last_finalized_overlay: Default::default(),
+		})
+	}
+
+	/// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window.
+	pub fn insert(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet<Key>) -> CommitSet<Key> {
+		let mut commit = CommitSet::default();
+		if self.levels.is_empty() && self.last_finalized.is_none() {
+			// assume that parent was finalized
+			let last_finalized = (parent_hash.clone(), number - 1);
+			commit.meta.inserted.push((to_meta_key(LAST_FINALIZED, &()), last_finalized.encode()));
+			self.last_finalized = Some(last_finalized);
+		} else if self.last_finalized.is_some() {
+			assert!(number >= self.front_block_number() && number < (self.front_block_number() + self.levels.len() as u64 + 1));
+			// check for valid parent if inserting on second level or higher
+			if number == self.front_block_number() {
+				assert!(self.last_finalized.as_ref().map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1));
+			} else {
+				assert!(self.parents.contains_key(&parent_hash));
+			}
+		}
+		let level = if self.levels.is_empty() || number == self.front_block_number() + self.levels.len() as u64 {
+			self.levels.push_back(Vec::new());
+			self.levels.back_mut().expect("can't be empty after insertion; qed")
+		} else {
+			let front_block_number = self.front_block_number();
+			self.levels.get_mut((number - front_block_number) as usize)
+				.expect("number is [front_block_number .. front_block_number + levels.len()) is asserted in precondition; qed")
+		};
+
+		let index = level.len() as u64;
+		let journal_key = to_journal_key(number, index);
+
+		let overlay = BlockOverlay {
+			hash: hash.clone(),
+			journal_key: journal_key.clone(),
+			values: changeset.inserted.iter().cloned().collect(),
+			deleted: changeset.deleted.clone(),
+		};
+		level.push(overlay);
+		self.parents.insert(hash.clone(), parent_hash.clone());
+		let journal_record = JournalRecord {
+			hash: hash.clone(),
+			parent_hash: parent_hash.clone(),
+			inserted: changeset.inserted,
+			deleted: changeset.deleted,
+		};
+		trace!(target: "state-db", "Inserted unfinalized changeset {}.{} ({} inserted, {} deleted)", number, index, journal_record.inserted.len(), journal_record.deleted.len());
+		let journal_record = journal_record.encode();
+		commit.meta.inserted.push((journal_key, journal_record));
+		commit
+	}
+
+	fn discard(
+		levels: &mut [Vec<BlockOverlay<BlockHash, Key>>],
+		parents: &mut HashMap<BlockHash, BlockHash>,
+		discarded_journals: &mut Vec<Vec<u8>>,
+		number: u64,
+		hash: &BlockHash,
+	) {
+		if let Some((level, sublevels)) = levels.split_first_mut() {
+			level.retain(|ref overlay| {
+				let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone();
+				if parent == *hash {
+					parents.remove(&overlay.hash);
+					discarded_journals.push(overlay.journal_key.clone());
+					Self::discard(sublevels, parents, discarded_journals, number + 1, &overlay.hash);
+					false
+				} else {
+					true
+				}
+			});
+		}
+	}
+
+	fn front_block_number(&self) -> u64 {
+		self.last_finalized.as_ref().map(|&(_, n)| n + 1).unwrap_or(0)
+	}
+
+	pub fn last_finalized_block_number(&self) -> u64 {
+		self.last_finalized.as_ref().map(|&(_, n)| n).unwrap_or(0)
+	}
+
+	/// This may be called when the last finalization commit was applied to the database.
+	pub fn clear_overlay(&mut self) {
+		self.last_finalized_overlay.clear();
+	}
+
+	/// Select a top-level root and finalized it. Discards all sibling subtrees and the root.
+	/// Returns a set of changes that need to be added to the DB.
+	pub fn finalize(&mut self, hash: &BlockHash) -> CommitSet<Key> {
+		trace!(target: "state-db", "Finalizing {:?}", hash);
+		let level = self.levels.pop_front().expect("no blocks to finalize");
+		let index = level.iter().position(|overlay| overlay.hash == *hash)
+			.expect("attempting to finalize unknown block");
+
+		let mut commit = CommitSet::default();
+		let mut discarded_journals = Vec::new();
+		for (i, overlay) in level.into_iter().enumerate() {
+			self.parents.remove(&overlay.hash);
+			if i == index {
+				self.last_finalized_overlay = overlay.values;
+				// that's the one we need to finalize
+				commit.data.inserted = self.last_finalized_overlay.iter().map(|(k, v)| (k.clone(), v.clone())).collect();
+				commit.data.deleted = overlay.deleted;
+			} else {
+				// TODO: borrow checker won't allow us to split out mutable refernces
+				// required for recursive processing. A more efficient implementaion
+				// that does not require converting to vector is possible
+				let mut vec: Vec<_> = self.levels.drain(..).collect();
+				Self::discard(&mut vec, &mut self.parents, &mut discarded_journals, 0, &overlay.hash);
+				self.levels.extend(vec.into_iter());
+			}
+			// cleanup journal entry
+			discarded_journals.push(overlay.journal_key);
+		}
+		commit.meta.deleted.append(&mut discarded_journals);
+		let last_finalized = (hash.clone(), self.front_block_number());
+		commit.meta.inserted.push((to_meta_key(LAST_FINALIZED, &()), last_finalized.encode()));
+		self.last_finalized = Some(last_finalized);
+		trace!(target: "state-db", "Discarded {} records", commit.meta.deleted.len());
+		commit
+	}
+
+	/// Get a value from the node overlay. This searches in every existing changeset.
+	pub fn get(&self, key: &Key) -> Option<DBValue> {
+		if let Some(value) = self.last_finalized_overlay.get(&key) {
+			return Some(value.clone());
+		}
+		for level in self.levels.iter() {
+			for overlay in level.iter() {
+				if let Some(value) = overlay.values.get(&key) {
+					return Some(value.clone());
+				}
+			}
+		}
+		None
+	}
+
+	/// Revert a single level. Returns commit set that deletes the journal or `None` if not possible.
+	pub fn revert_one(&mut self) -> Option<CommitSet<Key>> {
+		self.levels.pop_back().map(|level| {
+			let mut commit = CommitSet::default();
+			for overlay in level.into_iter() {
+				commit.meta.deleted.push(overlay.journal_key);
+				self.parents.remove(&overlay.hash);
+			}
+			commit
+		})
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::UnfinalizedOverlay;
+	use {ChangeSet};
+	use primitives::H256;
+	use test::{make_db, make_changeset};
+
+	fn contains(overlay: &UnfinalizedOverlay<H256, H256>, key: u64) -> bool {
+		overlay.get(&H256::from(key)) == Some(H256::from(key).to_vec())
+	}
+
+	#[test]
+	fn created_from_empty_db() {
+		let db = make_db(&[]);
+		let overlay: UnfinalizedOverlay<H256, H256> = UnfinalizedOverlay::new(&db).unwrap();
+		assert_eq!(overlay.last_finalized, None);
+		assert!(overlay.levels.is_empty());
+		assert!(overlay.parents.is_empty());
+	}
+
+	#[test]
+	#[should_panic]
+	fn finalize_empty_panics() {
+		let db = make_db(&[]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		overlay.finalize(&H256::default());
+	}
+
+	#[test]
+	#[should_panic]
+	fn insert_ahead_panics() {
+		let db = make_db(&[]);
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		overlay.insert(&h1, 2, &H256::default(), ChangeSet::default());
+		overlay.insert(&h2, 1, &h1, ChangeSet::default());
+	}
+
+	#[test]
+	#[should_panic]
+	fn insert_behind_panics() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let db = make_db(&[]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		overlay.insert(&h1, 1, &H256::default(), ChangeSet::default());
+		overlay.insert(&h2, 3, &h1, ChangeSet::default());
+	}
+
+	#[test]
+	#[should_panic]
+	fn insert_unknown_parent_panics() {
+		let db = make_db(&[]);
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		overlay.insert(&h1, 1, &H256::default(), ChangeSet::default());
+		overlay.insert(&h2, 2, &H256::default(), ChangeSet::default());
+	}
+
+	#[test]
+	#[should_panic]
+	fn finalize_unknown_panics() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let db = make_db(&[]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		overlay.insert(&h1, 1, &H256::default(), ChangeSet::default());
+		overlay.finalize(&h2);
+	}
+
+	#[test]
+	fn insert_finalize_one() {
+		let h1 = H256::random();
+		let mut db = make_db(&[1, 2]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		let changeset = make_changeset(&[3, 4], &[2]);
+		let insertion = overlay.insert(&h1, 1, &H256::default(), changeset.clone());
+		assert_eq!(insertion.data.inserted.len(), 0);
+		assert_eq!(insertion.data.deleted.len(), 0);
+		assert_eq!(insertion.meta.inserted.len(), 2);
+		assert_eq!(insertion.meta.deleted.len(), 0);
+		db.commit(&insertion);
+		let finalization = overlay.finalize(&h1);
+		assert_eq!(finalization.data.inserted.len(), changeset.inserted.len());
+		assert_eq!(finalization.data.deleted.len(), changeset.deleted.len());
+		assert_eq!(finalization.meta.inserted.len(), 1);
+		assert_eq!(finalization.meta.deleted.len(), 1);
+		db.commit(&finalization);
+		assert!(db.data_eq(&make_db(&[1, 3, 4])));
+	}
+
+	#[test]
+	fn restore_from_journal() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut db = make_db(&[1, 2]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		db.commit(&overlay.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])));
+		db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3])));
+		assert_eq!(db.meta.len(), 3);
+
+		let overlay2 = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		assert_eq!(overlay.levels, overlay2.levels);
+		assert_eq!(overlay.parents, overlay2.parents);
+		assert_eq!(overlay.last_finalized, overlay2.last_finalized);
+	}
+
+	#[test]
+	fn restore_from_journal_after_finalize() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut db = make_db(&[1, 2]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		db.commit(&overlay.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])));
+		db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3])));
+		db.commit(&overlay.finalize(&h1));
+		assert_eq!(overlay.levels.len(), 1);
+
+		let overlay2 = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		assert_eq!(overlay.levels, overlay2.levels);
+		assert_eq!(overlay.parents, overlay2.parents);
+		assert_eq!(overlay.last_finalized, overlay2.last_finalized);
+	}
+
+	#[test]
+	fn insert_finalize_two() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut db = make_db(&[1, 2, 3, 4]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		let changeset1 = make_changeset(&[5, 6], &[2]);
+		let changeset2 = make_changeset(&[7, 8], &[5, 3]);
+		db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1));
+		assert!(contains(&overlay, 5));
+		db.commit(&overlay.insert(&h2, 2, &h1, changeset2));
+		assert!(contains(&overlay, 7));
+		assert!(contains(&overlay, 5));
+		assert_eq!(overlay.levels.len(), 2);
+		assert_eq!(overlay.parents.len(), 2);
+		db.commit(&overlay.finalize(&h1));
+		assert_eq!(overlay.levels.len(), 1);
+		assert_eq!(overlay.parents.len(), 1);
+		assert!(contains(&overlay, 5));
+		overlay.clear_overlay();
+		assert!(!contains(&overlay, 5));
+		assert!(contains(&overlay, 7));
+		db.commit(&overlay.finalize(&h2));
+		overlay.clear_overlay();
+		assert_eq!(overlay.levels.len(), 0);
+		assert_eq!(overlay.parents.len(), 0);
+		assert!(db.data_eq(&make_db(&[1, 4, 6, 7, 8])));
+	}
+
+
+	#[test]
+	fn complex_tree() {
+		let mut db = make_db(&[]);
+
+		// - 1 - 1_1 - 1_1_1
+		//     \ 1_2 - 1_2_1
+		//           \ 1_2_2
+		//           \ 1_2_3
+		//
+		// - 2 - 2_1 - 2_1_1
+		//     \ 2_2
+		//
+		// 1_2_2 is the winner
+
+		let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[]));
+		let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[]));
+
+		let (h_1_1, c_1_1) = (H256::random(), make_changeset(&[11], &[]));
+		let (h_1_2, c_1_2) = (H256::random(), make_changeset(&[12], &[]));
+		let (h_2_1, c_2_1) = (H256::random(), make_changeset(&[21], &[]));
+		let (h_2_2, c_2_2) = (H256::random(), make_changeset(&[22], &[]));
+
+		let (h_1_1_1, c_1_1_1) = (H256::random(), make_changeset(&[111], &[]));
+		let (h_1_2_1, c_1_2_1) = (H256::random(), make_changeset(&[121], &[]));
+		let (h_1_2_2, c_1_2_2) = (H256::random(), make_changeset(&[122], &[]));
+		let (h_1_2_3, c_1_2_3) = (H256::random(), make_changeset(&[123], &[]));
+		let (h_2_1_1, c_2_1_1) = (H256::random(), make_changeset(&[211], &[]));
+
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1));
+
+		db.commit(&overlay.insert(&h_1_1, 2, &h_1, c_1_1));
+		db.commit(&overlay.insert(&h_1_2, 2, &h_1, c_1_2));
+
+		db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2));
+
+		db.commit(&overlay.insert(&h_2_1, 2, &h_2, c_2_1));
+		db.commit(&overlay.insert(&h_2_2, 2, &h_2, c_2_2));
+
+		db.commit(&overlay.insert(&h_1_1_1, 3, &h_1_1, c_1_1_1));
+		db.commit(&overlay.insert(&h_1_2_1, 3, &h_1_2, c_1_2_1));
+		db.commit(&overlay.insert(&h_1_2_2, 3, &h_1_2, c_1_2_2));
+		db.commit(&overlay.insert(&h_1_2_3, 3, &h_1_2, c_1_2_3));
+		db.commit(&overlay.insert(&h_2_1_1, 3, &h_2_1, c_2_1_1));
+
+		assert!(contains(&overlay, 2));
+		assert!(contains(&overlay, 11));
+		assert!(contains(&overlay, 21));
+		assert!(contains(&overlay, 111));
+		assert!(contains(&overlay, 122));
+		assert!(contains(&overlay, 211));
+		assert_eq!(overlay.levels.len(), 3);
+		assert_eq!(overlay.parents.len(), 11);
+		assert_eq!(overlay.last_finalized, Some((H256::default(), 0)));
+
+		// check if restoration from journal results in the same tree
+		let overlay2 = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		assert_eq!(overlay.levels, overlay2.levels);
+		assert_eq!(overlay.parents, overlay2.parents);
+		assert_eq!(overlay.last_finalized, overlay2.last_finalized);
+
+		// finalize 1. 2 and all its children should be discarded
+		db.commit(&overlay.finalize(&h_1));
+		overlay.clear_overlay();
+		assert_eq!(overlay.levels.len(), 2);
+		assert_eq!(overlay.parents.len(), 6);
+		assert!(!contains(&overlay, 1));
+		assert!(!contains(&overlay, 2));
+		assert!(!contains(&overlay, 21));
+		assert!(!contains(&overlay, 22));
+		assert!(!contains(&overlay, 211));
+		assert!(contains(&overlay, 111));
+
+		// finalize 1_2. 1_1 and all its children should be discarded
+		db.commit(&overlay.finalize(&h_1_2));
+		overlay.clear_overlay();
+		assert_eq!(overlay.levels.len(), 1);
+		assert_eq!(overlay.parents.len(), 3);
+		assert!(!contains(&overlay, 11));
+		assert!(!contains(&overlay, 111));
+		assert!(contains(&overlay, 121));
+		assert!(contains(&overlay, 122));
+		assert!(contains(&overlay, 123));
+
+		// finalize 1_2_2
+		db.commit(&overlay.finalize(&h_1_2_2));
+		overlay.clear_overlay();
+		assert_eq!(overlay.levels.len(), 0);
+		assert_eq!(overlay.parents.len(), 0);
+		assert!(db.data_eq(&make_db(&[1, 12, 122])));
+		assert_eq!(overlay.last_finalized, Some((h_1_2_2, 3)));
+	}
+
+	#[test]
+	fn insert_revert() {
+		let h1 = H256::random();
+		let h2 = H256::random();
+		let mut db = make_db(&[1, 2, 3, 4]);
+		let mut overlay = UnfinalizedOverlay::<H256, H256>::new(&db).unwrap();
+		assert!(overlay.revert_one().is_none());
+		let changeset1 = make_changeset(&[5, 6], &[2]);
+		let changeset2 = make_changeset(&[7, 8], &[5, 3]);
+		db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1));
+		db.commit(&overlay.insert(&h2, 2, &h1, changeset2));
+		assert!(contains(&overlay, 7));
+		db.commit(&overlay.revert_one().unwrap());
+		assert_eq!(overlay.parents.len(), 1);
+		assert!(contains(&overlay, 5));
+		assert!(!contains(&overlay, 7));
+		db.commit(&overlay.revert_one().unwrap());
+		assert_eq!(overlay.levels.len(), 0);
+		assert_eq!(overlay.parents.len(), 0);
+		assert!(overlay.revert_one().is_none());
+	}
+
+}
+