Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/db/src/cache/list_entry.rs

@@ -0,0 +1,161 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage entries.
+
+use client_api::error::Result as ClientResult;
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use codec::{Encode, Decode};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_storage::{Storage};
+
+/// Single list-based cache entry.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct Entry<Block: BlockT, T> {
+	/// first block, when this value became actual.
+	pub valid_from: ComplexBlockId<Block>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+/// Internal representation of the single list-based cache entry. The entry points to the
+/// previous entry in the cache, allowing us to traverse back in time in list-style.
+#[derive(Debug, Encode, Decode)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct StorageEntry<Block: BlockT, T: CacheItemT> {
+	/// None if valid from the beginning.
+	pub prev_valid_from: Option<ComplexBlockId<Block>>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
+	/// Returns Some if the entry should be updated with the new value.
+	pub fn try_update(&self, value: Option<T>) -> Option<StorageEntry<Block, T>> {
+		match value {
+			Some(value) => match self.value == value {
+				true => None,
+				false => Some(StorageEntry {
+					prev_valid_from: Some(self.valid_from.clone()),
+					value,
+				}),
+			},
+			None => None,
+		}
+	}
+
+	/// Wrapper that calls search_before to get range where the given block fits.
+	pub fn search_best_range_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
+		Ok(self.search_best_before(storage, block)?
+			.map(|(entry, next)| (entry.valid_from, next)))
+	}
+
+	/// Searches the list, ending with THIS entry for the best entry preceeding (or at)
+	/// given block number.
+	/// If the entry is found, result is the entry and the block id of next entry (if exists).
+	/// NOTE that this function does not check that the passed block is actually linked to
+	/// the blocks it found.
+	pub fn search_best_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(Entry<Block, T>, Option<ComplexBlockId<Block>>)>> {
+		// we're looking for the best value
+		let mut next = None;
+		let mut current = self.valid_from.clone();
+		if block >= self.valid_from.number {
+			let value = self.value.clone();
+			return Ok(Some((Entry { valid_from: current, value }, next)));
+		}
+
+		// else - travel back in time
+		loop {
+			let entry = storage.require_entry(&current)?;
+			if block >= current.number {
+				return Ok(Some((Entry { valid_from: current, value: entry.value }, next)));
+			}
+
+			next = Some(current);
+			current = match entry.prev_valid_from {
+				Some(prev_valid_from) => prev_valid_from,
+				None => return Ok(None),
+			};
+		}
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
+	/// Converts storage entry into an entry, valid from given block.
+	pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
+		Entry {
+			valid_from,
+			value: self.value,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::cache::list_cache::tests::test_id;
+	use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
+	use super::*;
+
+	#[test]
+	fn entry_try_update_works() {
+		// when trying to update with None value
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(1), value: 42 }.try_update(None), None);
+		// when trying to update with the same Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
+		// when trying to update with different Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
+			Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }));
+	}
+
+	#[test]
+	fn entry_search_best_before_fails() {
+		// when storage returns error
+		assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
+			.search_best_before(&FaultyStorage, 50).is_err());
+	}
+
+	#[test]
+	fn entry_search_best_before_works() {
+		// when block is better than our best block
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new(), 150).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)));
+		// when block is found between two entries
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }),
+			75).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))));
+		// when block is not found
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
+			30).unwrap(),
+		None);
+	}
+}

substrate/client/db/src/cache/list_storage.rs

@@ -0,0 +1,386 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage definition and implementation.
+
+use std::sync::Arc;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::error::{Error as ClientError, Result as ClientResult};
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+use crate::utils::{self, db_err, meta_keys};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_cache::{CommitOperation, Fork};
+use crate::cache::list_entry::{Entry, StorageEntry};
+
+/// Single list-cache metadata.
+#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct Metadata<Block: BlockT> {
+	/// Block at which best finalized entry is stored.
+	pub finalized: Option<ComplexBlockId<Block>>,
+	/// A set of blocks at which best unfinalized entries are stored.
+	pub unfinalized: Vec<ComplexBlockId<Block>>,
+}
+
+/// Readonly list-cache storage trait.
+pub trait Storage<Block: BlockT, T: CacheItemT> {
+	/// Reads hash of the block at given number.
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>>;
+
+	/// Reads header of the block with given hash.
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>>;
+
+	/// Reads cache metadata: best finalized entry (if some) and the list.
+	fn read_meta(&self) -> ClientResult<Metadata<Block>>;
+
+	/// Reads cache entry from the storage.
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>>;
+
+	/// Reads referenced (and thus existing) cache entry from the storage.
+	fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
+		self.read_entry(at)
+			.and_then(|entry| entry
+				.ok_or_else(|| ClientError::from(
+					ClientError::Backend(format!("Referenced cache entry at {:?} is not found", at)))))
+	}
+}
+
+/// List-cache storage transaction.
+pub trait StorageTransaction<Block: BlockT, T: CacheItemT> {
+	/// Insert storage entry at given block.
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>);
+
+	/// Delete storage entry at given block.
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>);
+
+	/// Update metadata of the cache.
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	);
+}
+
+/// A set of columns used by the DbStorage.
+#[derive(Debug)]
+pub struct DbColumns {
+	/// Column holding cache meta.
+	pub meta: Option<u32>,
+	/// Column holding the mapping of { block number => block hash } for blocks of the best chain.
+	pub key_lookup: Option<u32>,
+	/// Column holding the mapping of { block hash => block header }.
+	pub header: Option<u32>,
+	/// Column holding cache entries.
+	pub cache: Option<u32>,
+}
+
+/// Database-backed list cache storage.
+pub struct DbStorage {
+	name: Vec<u8>,
+	meta_key: Vec<u8>,
+	db: Arc<dyn KeyValueDB>,
+	columns: DbColumns,
+}
+
+impl DbStorage {
+	/// Create new database-backed list cache storage.
+	pub fn new(name: Vec<u8>, db: Arc<dyn KeyValueDB>, columns: DbColumns) -> Self {
+		let meta_key = meta::key(&name);
+		DbStorage { name, meta_key, db, columns }
+	}
+
+	/// Get reference to the database.
+	pub fn db(&self) -> &Arc<dyn KeyValueDB> { &self.db }
+
+	/// Get reference to the database columns.
+	pub fn columns(&self) -> &DbColumns { &self.columns }
+
+	/// Encode block id for storing as a key in cache column.
+	/// We append prefix to the actual encoding to allow several caches
+	/// store entries in the same column.
+	pub fn encode_block_id<Block: BlockT>(&self, block: &ComplexBlockId<Block>) -> Vec<u8> {
+		let mut encoded = self.name.clone();
+		encoded.extend(block.hash.as_ref());
+		encoded
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Number(at))
+			.map(|maybe_header| maybe_header.map(|header| header.hash()))
+	}
+
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Hash(*at))
+	}
+
+	fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+		self.db.get(self.columns.meta, &self.meta_key)
+			.map_err(db_err)
+			.and_then(|meta| match meta {
+				Some(meta) => meta::decode(&*meta),
+				None => Ok(Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				}),
+			})
+	}
+
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+		self.db.get(self.columns.cache, &self.encode_block_id(at))
+			.map_err(db_err)
+			.and_then(|entry| match entry {
+				Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
+					.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
+					.map(Some),
+				None => Ok(None),
+			})
+	}
+}
+
+/// Database-backed list cache storage transaction.
+pub struct DbStorageTransaction<'a> {
+	storage: &'a DbStorage,
+	tx: &'a mut DBTransaction,
+}
+
+impl<'a> DbStorageTransaction<'a> {
+	/// Create new database transaction.
+	pub fn new(storage: &'a DbStorage, tx: &'a mut DBTransaction) -> Self {
+		DbStorageTransaction { storage, tx }
+	}
+}
+
+impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
+		self.tx.put(self.storage.columns.cache, &self.storage.encode_block_id(at), &entry.encode());
+	}
+
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+		self.tx.delete(self.storage.columns.cache, &self.storage.encode_block_id(at));
+	}
+
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	) {
+		self.tx.put(
+			self.storage.columns.meta,
+			&self.storage.meta_key,
+			&meta::encode(best_finalized_entry, unfinalized, operation));
+	}
+}
+
+/// Metadata related functions.
+mod meta {
+	use super::*;
+
+	/// Convert cache name into cache metadata key.
+	pub fn key(name: &[u8]) -> Vec<u8> {
+		let mut key_name = meta_keys::CACHE_META_PREFIX.to_vec();
+		key_name.extend_from_slice(name);
+		key_name
+	}
+
+	/// Encode cache metadata 'applying' commit operation before encoding.
+	pub fn encode<Block: BlockT, T: CacheItemT>(
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		op: &CommitOperation<Block, T>
+	) -> Vec<u8> {
+		let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
+		let mut unfinalized = unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
+
+		match op {
+			CommitOperation::AppendNewBlock(_, _) => (),
+			CommitOperation::AppendNewEntry(index, ref entry) => {
+				unfinalized[*index] = &entry.valid_from;
+			},
+			CommitOperation::AddNewFork(ref entry) => {
+				unfinalized.push(&entry.valid_from);
+			},
+			CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
+				finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
+				for fork_index in forks.iter().rev() {
+					unfinalized.remove(*fork_index);
+				}
+			},
+			CommitOperation::BlockReverted(ref forks) => {
+				for (fork_index, updated_fork) in forks.iter().rev() {
+					match updated_fork {
+						Some(updated_fork) => unfinalized[*fork_index] = &updated_fork.head().valid_from,
+						None => { unfinalized.remove(*fork_index); },
+					}
+				}
+			},
+		}
+
+		(finalized, unfinalized).encode()
+	}
+
+	/// Decode meta information.
+	pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
+		let input = &mut &*encoded;
+		let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+		let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+
+		Ok(Metadata { finalized, unfinalized })
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use std::collections::{HashMap, HashSet};
+	use super::*;
+
+	pub struct FaultyStorage;
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for FaultyStorage {
+		fn read_id(&self, _at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_header(&self, _at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_entry(&self, _at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+	}
+
+	pub struct DummyStorage<Block: BlockT, T: CacheItemT> {
+		meta: Metadata<Block>,
+		ids: HashMap<NumberFor<Block>, Block::Hash>,
+		headers: HashMap<Block::Hash, Block::Header>,
+		entries: HashMap<Block::Hash, StorageEntry<Block, T>>,
+	}
+
+	impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
+		pub fn new() -> Self {
+			DummyStorage {
+				meta: Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				},
+				ids: HashMap::new(),
+				headers: HashMap::new(),
+				entries: HashMap::new(),
+			}
+		}
+
+		pub fn with_meta(mut self, finalized: Option<ComplexBlockId<Block>>, unfinalized: Vec<ComplexBlockId<Block>>) -> Self {
+			self.meta.finalized = finalized;
+			self.meta.unfinalized = unfinalized;
+			self
+		}
+
+		pub fn with_id(mut self, at: NumberFor<Block>, id: Block::Hash) -> Self {
+			self.ids.insert(at, id);
+			self
+		}
+
+		pub fn with_header(mut self, header: Block::Header) -> Self {
+			self.headers.insert(header.hash(), header);
+			self
+		}
+
+		pub fn with_entry(mut self, at: ComplexBlockId<Block>, entry: StorageEntry<Block, T>) -> Self {
+			self.entries.insert(at.hash, entry);
+			self
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DummyStorage<Block, T> {
+		fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Ok(self.ids.get(&at).cloned())
+		}
+
+		fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Ok(self.headers.get(&at).cloned())
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Ok(self.meta.clone())
+		}
+
+		fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Ok(self.entries.get(&at.hash).cloned())
+		}
+	}
+
+	pub struct DummyTransaction<Block: BlockT> {
+		updated_meta: Option<Metadata<Block>>,
+		inserted_entries: HashSet<Block::Hash>,
+		removed_entries: HashSet<Block::Hash>,
+	}
+
+	impl<Block: BlockT> DummyTransaction<Block> {
+		pub fn new() -> Self {
+			DummyTransaction {
+				updated_meta: None,
+				inserted_entries: HashSet::new(),
+				removed_entries: HashSet::new(),
+			}
+		}
+
+		pub fn inserted_entries(&self) -> &HashSet<Block::Hash> {
+			&self.inserted_entries
+		}
+
+		pub fn removed_entries(&self) -> &HashSet<Block::Hash> {
+			&self.removed_entries
+		}
+
+		pub fn updated_meta(&self) -> &Option<Metadata<Block>> {
+			&self.updated_meta
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
+		fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, _entry: &StorageEntry<Block, T>) {
+			self.inserted_entries.insert(at.hash);
+		}
+
+		fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+			self.removed_entries.insert(at.hash);
+		}
+
+		fn update_meta(
+			&mut self,
+			best_finalized_entry: Option<&Entry<Block, T>>,
+			unfinalized: &[Fork<Block, T>],
+			operation: &CommitOperation<Block, T>,
+		) {
+			self.updated_meta = Some(meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap());
+		}
+	}
+}

substrate/client/db/src/cache/mod.rs

@@ -0,0 +1,371 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! DB-backed cache of blockchain data.
+
+use std::{sync::Arc, collections::HashMap};
+use parking_lot::RwLock;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
+use client_api::error::Result as ClientResult;
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
+use crate::utils::{self, COLUMN_META, db_err};
+
+use self::list_cache::{ListCache, PruningStrategy};
+
+mod list_cache;
+mod list_entry;
+mod list_storage;
+
+/// Minimal post-finalization age of finalized blocks before they'll pruned.
+const PRUNE_DEPTH: u32 = 1024;
+
+/// The type of entry that is inserted to the cache.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum EntryType {
+	/// Non-final entry.
+	NonFinal,
+	/// Final entry.
+	Final,
+	/// Genesis entry (inserted during cache initialization).
+	Genesis,
+}
+
+/// Block identifier that holds both hash and number.
+#[derive(Clone, Debug, Encode, Decode, PartialEq)]
+pub struct ComplexBlockId<Block: BlockT> {
+	hash: Block::Hash,
+	number: NumberFor<Block>,
+}
+
+impl<Block: BlockT> ComplexBlockId<Block> {
+	/// Create new complex block id.
+	pub fn new(hash: Block::Hash, number: NumberFor<Block>) -> Self {
+		ComplexBlockId { hash, number }
+	}
+}
+
+impl<Block: BlockT> ::std::cmp::PartialOrd for ComplexBlockId<Block> {
+	fn partial_cmp(&self, other: &ComplexBlockId<Block>) -> Option<::std::cmp::Ordering> {
+		self.number.partial_cmp(&other.number)
+	}
+}
+
+/// All cache items must implement this trait.
+pub trait CacheItemT: Clone + Decode + Encode + PartialEq {}
+
+impl<T> CacheItemT for T where T: Clone + Decode + Encode + PartialEq {}
+
+/// Database-backed blockchain data cache.
+pub struct DbCache<Block: BlockT> {
+	cache_at: HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	db: Arc<dyn KeyValueDB>,
+	key_lookup_column: Option<u32>,
+	header_column: Option<u32>,
+	authorities_column: Option<u32>,
+	genesis_hash: Block::Hash,
+	best_finalized_block: ComplexBlockId<Block>,
+}
+
+impl<Block: BlockT> DbCache<Block> {
+	/// Create new cache.
+	pub fn new(
+		db: Arc<dyn KeyValueDB>,
+		key_lookup_column: Option<u32>,
+		header_column: Option<u32>,
+		authorities_column: Option<u32>,
+		genesis_hash: Block::Hash,
+		best_finalized_block: ComplexBlockId<Block>,
+	) -> Self {
+		Self {
+			cache_at: HashMap::new(),
+			db,
+			key_lookup_column,
+			header_column,
+			authorities_column,
+			genesis_hash,
+			best_finalized_block,
+		}
+	}
+
+	/// Set genesis block hash.
+	pub fn set_genesis_hash(&mut self, genesis_hash: Block::Hash) {
+		self.genesis_hash = genesis_hash;
+	}
+
+	/// Begin cache transaction.
+	pub fn transaction<'a>(&'a mut self, tx: &'a mut DBTransaction) -> DbCacheTransaction<'a, Block> {
+		DbCacheTransaction {
+			cache: self,
+			tx,
+			cache_at_op: HashMap::new(),
+			best_finalized_block: None,
+		}
+	}
+
+	/// Run post-commit cache operations.
+	pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
+		for (name, op) in ops.cache_at_op.into_iter() {
+			self.get_cache(name).on_transaction_commit(op);
+		}
+		if let Some(best_finalized_block) = ops.best_finalized_block {
+			self.best_finalized_block = best_finalized_block;
+		}
+	}
+
+	/// Creates `ListCache` with the given name or returns a reference to the existing.
+	fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+		get_cache_helper(
+			&mut self.cache_at,
+			name,
+			&self.db,
+			self.key_lookup_column,
+			self.header_column,
+			self.authorities_column,
+			&self.best_finalized_block
+		)
+	}
+}
+
+// This helper is needed because otherwise the borrow checker will require to
+// clone all parameters outside of the closure.
+fn get_cache_helper<'a, Block: BlockT>(
+	cache_at: &'a mut HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	name: CacheKeyId,
+	db: &Arc<dyn KeyValueDB>,
+	key_lookup: Option<u32>,
+	header: Option<u32>,
+	cache: Option<u32>,
+	best_finalized_block: &ComplexBlockId<Block>,
+) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+	cache_at.entry(name).or_insert_with(|| {
+		ListCache::new(
+			self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
+				self::list_storage::DbColumns {
+					meta: COLUMN_META,
+					key_lookup,
+					header,
+					cache,
+				},
+			),
+			cache_pruning_strategy(name),
+			best_finalized_block.clone(),
+		)
+	})
+}
+
+/// Cache operations that are to be committed after database transaction is committed.
+pub struct DbCacheTransactionOps<Block: BlockT> {
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+/// Database-backed blockchain data cache transaction valid for single block import.
+pub struct DbCacheTransaction<'a, Block: BlockT> {
+	cache: &'a mut DbCache<Block>,
+	tx: &'a mut DBTransaction,
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
+	/// Convert transaction into post-commit operations set.
+	pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
+		DbCacheTransactionOps {
+			cache_at_op: self.cache_at_op,
+			best_finalized_block: self.best_finalized_block,
+		}
+	}
+
+	/// When new block is inserted into database.
+	pub fn on_block_insert(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>,
+		data_at: HashMap<CacheKeyId, Vec<u8>>,
+		entry_type: EntryType,
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		// prepare list of caches that are not update
+		// (we might still need to do some cache maintenance in this case)
+		let missed_caches = self.cache.cache_at.keys()
+			.filter(|cache| !data_at.contains_key(*cache))
+			.cloned()
+			.collect::<Vec<_>>();
+
+		let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), client_api::error::Error> {
+			let cache = self.cache.get_cache(name);
+			let op = cache.on_block_insert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx,
+				),
+				parent.clone(),
+				block.clone(),
+				value,
+				entry_type,
+			)?;
+			if let Some(op) = op {
+				self.cache_at_op.insert(name, op);
+			}
+			Ok(())
+		};
+
+		data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?;
+		missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
+
+		match entry_type {
+			EntryType::Final | EntryType::Genesis =>
+				self.best_finalized_block = Some(block),
+			EntryType::NonFinal => (),
+		}
+
+		Ok(self)
+	}
+
+	/// When previously inserted block is finalized.
+	pub fn on_block_finalize(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		for (name, cache_at) in self.cache.cache_at.iter() {
+			let op = cache_at.on_block_finalize(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache_at.storage(),
+					&mut self.tx
+				),
+				parent.clone(),
+				block.clone(),
+			)?;
+
+			if let Some(op) = op {
+				self.cache_at_op.insert(name.to_owned(), op);
+			}
+		}
+
+		self.best_finalized_block = Some(block);
+
+		Ok(self)
+	}
+
+	/// When block is reverted.
+	pub fn on_block_revert(
+		mut self,
+		reverted_block: &ComplexBlockId<Block>,
+	) -> ClientResult<Self> {
+		for (name, cache) in self.cache.cache_at.iter() {
+			let op = cache.on_block_revert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx
+				),
+				reverted_block,
+			)?;
+
+			assert!(!self.cache_at_op.contains_key(name));
+			self.cache_at_op.insert(name.to_owned(), op);
+		}
+
+		Ok(self)
+	}
+}
+
+/// Synchronous implementation of database-backed blockchain data cache.
+pub struct DbCacheSync<Block: BlockT>(pub RwLock<DbCache<Block>>);
+
+impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
+	fn initialize(&self, key: &CacheKeyId, data: Vec<u8>) -> ClientResult<()> {
+		let mut cache = self.0.write();
+		let genesis_hash = cache.genesis_hash;
+		let cache_contents = vec![(*key, data)].into_iter().collect();
+		let db = cache.db.clone();
+		let mut dbtx = DBTransaction::new();
+		let tx = cache.transaction(&mut dbtx);
+		let tx = tx.on_block_insert(
+			ComplexBlockId::new(Default::default(), Zero::zero()),
+			ComplexBlockId::new(genesis_hash, Zero::zero()),
+			cache_contents,
+			EntryType::Genesis,
+		)?;
+		let tx_ops = tx.into_ops();
+		db.write(dbtx).map_err(db_err)?;
+		cache.commit(tx_ops);
+		Ok(())
+	}
+
+	fn get_at(
+		&self,
+		key: &CacheKeyId,
+		at: &BlockId<Block>,
+	) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
+		let mut cache = self.0.write();
+		let storage = cache.get_cache(*key).storage();
+		let db = storage.db();
+		let columns = storage.columns();
+		let at = match *at {
+			BlockId::Hash(hash) => {
+				let header = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Hash(hash.clone())).ok()??;
+				ComplexBlockId::new(hash, *header.number())
+			},
+			BlockId::Number(number) => {
+				let hash = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Number(number.clone())).ok()??.hash();
+				ComplexBlockId::new(hash, number)
+			},
+		};
+
+		cache.cache_at
+			.get(key)?
+			.value_at_block(&at)
+			.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
+				(
+					(begin_block.number, begin_block.hash),
+					end_block.map(|end_block| (end_block.number, end_block.hash)),
+					value,
+				)))
+			.ok()?
+	}
+}
+
+/// Get pruning strategy for given cache.
+fn cache_pruning_strategy<N: From<u32>>(cache: CacheKeyId) -> PruningStrategy<N> {
+	// the cache is mostly used to store data from consensus engines
+	// this kind of data is only required for non-finalized blocks
+	// => by default we prune finalized cached entries
+
+	match cache {
+		// we need to keep changes tries configurations forever (or at least until changes tries,
+		// that were built using this configuration, are pruned) to make it possible to refer
+		// to old changes tries
+		well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune,
+		_ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()),
+	}
+}

substrate/client/db/src/children.rs

@@ -0,0 +1,121 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Functionality for reading and storing children hashes from db.
+
+use kvdb::{KeyValueDB, DBTransaction};
+use codec::{Encode, Decode};
+use client_api::error;
+use std::hash::Hash;
+
+
+/// Returns the hashes of the children blocks of the block with `parent_hash`.
+pub fn read_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+	V: Eq + Hash + Clone + Encode + Decode,
+>(db: &dyn KeyValueDB, column: Option<u32>, prefix: &[u8], parent_hash: K) -> error::Result<Vec<V>> {
+	let mut buf = prefix.to_vec();
+	parent_hash.using_encoded(|s| buf.extend(s));
+
+	let raw_val_opt = match db.get(column, &buf[..]) {
+		Ok(raw_val_opt) => raw_val_opt,
+		Err(_) => return Err(error::Error::Backend("Error reading value from database".into())),
+	};
+
+	let raw_val = match raw_val_opt {
+		Some(val) => val,
+		None => return Ok(Vec::new()),
+	};
+
+	let children: Vec<V> = match Decode::decode(&mut &raw_val[..]) {
+		Ok(children) => children,
+		Err(_) => return Err(error::Error::Backend("Error decoding children".into())),
+	};
+
+	Ok(children)
+}
+
+/// Insert the key-value pair (`parent_hash`, `children_hashes`) in the transaction.
+/// Any existing value is overwritten upon write.
+pub fn write_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+	V: Eq + Hash + Clone + Encode + Decode,
+>(
+	tx: &mut DBTransaction,
+	column: Option<u32>,
+	prefix: &[u8],
+	parent_hash: K,
+	children_hashes: V,
+) {
+	let mut key = prefix.to_vec();
+	parent_hash.using_encoded(|s| key.extend(s));
+	tx.put_vec(column, &key[..], children_hashes.encode());
+}
+
+/// Prepare transaction to remove the children of `parent_hash`.
+pub fn remove_children<
+	K: Eq + Hash + Clone + Encode + Decode,
+>(
+	tx: &mut DBTransaction,
+	column: Option<u32>,
+	prefix: &[u8],
+	parent_hash: K,
+) {
+	let mut key = prefix.to_vec();
+	parent_hash.using_encoded(|s| key.extend(s));
+	tx.delete(column, &key[..]);
+}
+
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn children_write_read_remove() {
+		const PREFIX: &[u8] = b"children";
+		let db = ::kvdb_memorydb::create(0);
+
+		let mut tx = DBTransaction::new();
+
+		let mut children1 = Vec::new();
+		children1.push(1_3);
+		children1.push(1_5);
+		write_children(&mut tx, None, PREFIX, 1_1, children1);
+
+		let mut children2 = Vec::new();
+		children2.push(1_4);
+		children2.push(1_6);
+		write_children(&mut tx, None, PREFIX, 1_2, children2);
+
+		db.write(tx.clone()).unwrap();
+
+		let r1: Vec<u32> = read_children(&db, None, PREFIX, 1_1).unwrap();
+		let r2: Vec<u32> = read_children(&db, None, PREFIX, 1_2).unwrap();
+
+		assert_eq!(r1, vec![1_3, 1_5]);
+		assert_eq!(r2, vec![1_4, 1_6]);
+
+		remove_children(&mut tx, None, PREFIX, 1_2);
+		db.write(tx).unwrap();
+
+		let r1: Vec<u32> = read_children(&db, None, PREFIX, 1_1).unwrap();
+		let r2: Vec<u32> = read_children(&db, None, PREFIX, 1_2).unwrap();
+
+		assert_eq!(r1, vec![1_3, 1_5]);
+		assert_eq!(r2.len(), 0);
+	}
+}

substrate/client/db/src/offchain.rs

@@ -0,0 +1,148 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! RocksDB-based offchain workers local storage.
+
+use std::{
+	collections::HashMap,
+	sync::Arc,
+};
+
+use crate::columns;
+use kvdb::KeyValueDB;
+use parking_lot::Mutex;
+
+/// Offchain local storage
+#[derive(Clone)]
+pub struct LocalStorage {
+	db: Arc<dyn KeyValueDB>,
+	locks: Arc<Mutex<HashMap<Vec<u8>, Arc<Mutex<()>>>>>,
+}
+
+impl std::fmt::Debug for LocalStorage {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.debug_struct("LocalStorage")
+			.finish()
+	}
+}
+
+impl LocalStorage {
+	/// Create new offchain storage for tests (backed by memorydb)
+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test() -> Self {
+		let db = Arc::new(::kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
+		Self::new(db as _)
+	}
+
+	/// Create offchain local storage with given `KeyValueDB` backend.
+	pub fn new(db: Arc<dyn KeyValueDB>) -> Self {
+		Self {
+			db,
+			locks: Default::default(),
+		}
+	}
+}
+
+impl client_api::OffchainStorage for LocalStorage {
+	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		let mut tx = self.db.transaction();
+		tx.put(columns::OFFCHAIN, &key, value);
+
+		if let Err(e) = self.db.write(tx) {
+			log::warn!("Error writing to the offchain DB: {:?}", e);
+		}
+	}
+
+	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
+		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
+		self.db.get(columns::OFFCHAIN, &key)
+			.ok()
+			.and_then(|x| x)
+			.map(|v| v.to_vec())
+	}
+
+	fn compare_and_set(
+		&mut self,
+		prefix: &[u8],
+		item_key: &[u8],
+		old_value: Option<&[u8]>,
+		new_value: &[u8],
+	) -> bool {
+		let key: Vec<u8> = prefix.iter().chain(item_key).cloned().collect();
+		let key_lock = {
+			let mut locks = self.locks.lock();
+			locks.entry(key.clone()).or_default().clone()
+		};
+
+		let is_set;
+		{
+			let _key_guard = key_lock.lock();
+			let val = self.db.get(columns::OFFCHAIN, &key)
+				.ok()
+				.and_then(|x| x);
+			is_set = val.as_ref().map(|x| &**x) == old_value;
+
+			if is_set {
+				self.set(prefix, item_key, new_value)
+			}
+		}
+
+		// clean the lock map if we're the only entry
+		let mut locks = self.locks.lock();
+		{
+			drop(key_lock);
+			let key_lock = locks.get_mut(&key);
+			if let Some(_) = key_lock.and_then(Arc::get_mut) {
+				locks.remove(&key);
+			}
+		}
+		is_set
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use client_api::OffchainStorage;
+
+	#[test]
+	fn should_compare_and_set_and_clear_the_locks_map() {
+		let mut storage = LocalStorage::new_test();
+		let prefix = b"prefix";
+		let key = b"key";
+		let value = b"value";
+
+		storage.set(prefix, key, value);
+		assert_eq!(storage.get(prefix, key), Some(value.to_vec()));
+
+		assert_eq!(storage.compare_and_set(prefix, key, Some(value), b"asd"), true);
+		assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
+		assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
+	}
+
+	#[test]
+	fn should_compare_and_set_on_empty_field() {
+		let mut storage = LocalStorage::new_test();
+		let prefix = b"prefix";
+		let key = b"key";
+
+		assert_eq!(storage.compare_and_set(prefix, key, None, b"asd"), true);
+		assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
+		assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
+	}
+
+}

substrate/client/db/src/utils.rs

@@ -0,0 +1,356 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Db-based backend utility structures and functions, used by both
+//! full and light storages.
+
+use std::sync::Arc;
+use std::{io, convert::TryInto};
+
+use kvdb::{KeyValueDB, DBTransaction};
+#[cfg(feature = "kvdb-rocksdb")]
+use kvdb_rocksdb::{Database, DatabaseConfig};
+use log::debug;
+
+use client;
+use codec::Decode;
+use trie::DBValue;
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{
+	Block as BlockT, Header as HeaderT, Zero,
+	UniqueSaturatedFrom, UniqueSaturatedInto,
+};
+use crate::{DatabaseSettings, DatabaseSettingsSrc};
+
+/// Number of columns in the db. Must be the same for both full && light dbs.
+/// Otherwise RocksDb will fail to open database && check its type.
+pub const NUM_COLUMNS: u32 = 10;
+/// Meta column. The set of keys in the column is shared by full && light storages.
+pub const COLUMN_META: Option<u32> = Some(0);
+
+/// Keys of entries in COLUMN_META.
+pub mod meta_keys {
+	/// Type of storage (full or light).
+	pub const TYPE: &[u8; 4] = b"type";
+	/// Best block key.
+	pub const BEST_BLOCK: &[u8; 4] = b"best";
+	/// Last finalized block key.
+	pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
+	/// Meta information prefix for list-based caches.
+	pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
+	/// Genesis block hash.
+	pub const GENESIS_HASH: &[u8; 3] = b"gen";
+	/// Leaves prefix list key.
+	pub const LEAF_PREFIX: &[u8; 4] = b"leaf";
+	/// Children prefix list key.
+	pub const CHILDREN_PREFIX: &[u8; 8] = b"children";
+}
+
+/// Database metadata.
+#[derive(Debug)]
+pub struct Meta<N, H> {
+	/// Hash of the best known block.
+	pub best_hash: H,
+	/// Number of the best known block.
+	pub best_number: N,
+	/// Hash of the best finalized block.
+	pub finalized_hash: H,
+	/// Number of the best finalized block.
+	pub finalized_number: N,
+	/// Hash of the genesis block.
+	pub genesis_hash: H,
+}
+
+/// A block lookup key: used for canonical lookup from block number to hash
+pub type NumberIndexKey = [u8; 4];
+
+/// Convert block number into short lookup key (LE representation) for
+/// blocks that are in the canonical chain.
+///
+/// In the current database schema, this kind of key is only used for
+/// lookups into an index, NOT for storing header data or others.
+pub fn number_index_key<N: TryInto<u32>>(n: N) -> client::error::Result<NumberIndexKey> {
+	let n = n.try_into().map_err(|_|
+		client::error::Error::Backend("Block number cannot be converted to u32".into())
+	)?;
+
+	Ok([
+		(n >> 24) as u8,
+		((n >> 16) & 0xff) as u8,
+		((n >> 8) & 0xff) as u8,
+		(n & 0xff) as u8
+	])
+}
+
+/// Convert number and hash into long lookup key for blocks that are
+/// not in the canonical chain.
+pub fn number_and_hash_to_lookup_key<N, H>(
+	number: N,
+	hash: H,
+) -> client::error::Result<Vec<u8>>	where
+	N: TryInto<u32>,
+	H: AsRef<[u8]>,
+{
+	let mut lookup_key = number_index_key(number)?.to_vec();
+	lookup_key.extend_from_slice(hash.as_ref());
+	Ok(lookup_key)
+}
+
+/// Convert block lookup key into block number.
+/// all block lookup keys start with the block number.
+pub fn lookup_key_to_number<N>(key: &[u8]) -> client::error::Result<N> where
+	N: From<u32>
+{
+	if key.len() < 4 {
+		return Err(client::error::Error::Backend("Invalid block key".into()));
+	}
+	Ok((key[0] as u32) << 24
+		| (key[1] as u32) << 16
+		| (key[2] as u32) << 8
+		| (key[3] as u32)).map(Into::into)
+}
+
+/// Delete number to hash mapping in DB transaction.
+pub fn remove_number_to_key_mapping<N: TryInto<u32>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+) -> client::error::Result<()> {
+	transaction.delete(key_lookup_col, number_index_key(number)?.as_ref());
+	Ok(())
+}
+
+/// Remove key mappings.
+pub fn remove_key_mappings<N: TryInto<u32>, H: AsRef<[u8]>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	remove_number_to_key_mapping(transaction, key_lookup_col, number)?;
+	transaction.delete(key_lookup_col, hash.as_ref());
+	Ok(())
+}
+
+/// Place a number mapping into the database. This maps number to current perceived
+/// block hash at that position.
+pub fn insert_number_to_key_mapping<N: TryInto<u32> + Clone, H: AsRef<[u8]>>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	transaction.put_vec(
+		key_lookup_col,
+		number_index_key(number.clone())?.as_ref(),
+		number_and_hash_to_lookup_key(number, hash)?,
+	);
+	Ok(())
+}
+
+/// Insert a hash to key mapping in the database.
+pub fn insert_hash_to_key_mapping<N: TryInto<u32>, H: AsRef<[u8]> + Clone>(
+	transaction: &mut DBTransaction,
+	key_lookup_col: Option<u32>,
+	number: N,
+	hash: H,
+) -> client::error::Result<()> {
+	transaction.put_vec(
+		key_lookup_col,
+		hash.clone().as_ref(),
+		number_and_hash_to_lookup_key(number, hash)?,
+	);
+	Ok(())
+}
+
+/// Convert block id to block lookup key.
+/// block lookup key is the DB-key header, block and justification are stored under.
+/// looks up lookup key by hash from DB as necessary.
+pub fn block_id_to_lookup_key<Block>(
+	db: &dyn KeyValueDB,
+	key_lookup_col: Option<u32>,
+	id: BlockId<Block>
+) -> Result<Option<Vec<u8>>, client::error::Error> where
+	Block: BlockT,
+	::sr_primitives::traits::NumberFor<Block>: UniqueSaturatedFrom<u64> + UniqueSaturatedInto<u64>,
+{
+	let res = match id {
+		BlockId::Number(n) => db.get(
+			key_lookup_col,
+			number_index_key(n)?.as_ref(),
+		),
+		BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()),
+	};
+
+	res.map(|v| v.map(|v| v.into_vec())).map_err(db_err)
+}
+
+/// Maps database error to client error
+pub fn db_err(err: io::Error) -> client::error::Error {
+	client::error::Error::Backend(format!("{}", err))
+}
+
+/// Open RocksDB database.
+pub fn open_database(
+	config: &DatabaseSettings,
+	col_meta: Option<u32>,
+	db_type: &str
+) -> client::error::Result<Arc<dyn KeyValueDB>> {
+	let db: Arc<dyn KeyValueDB> = match &config.source {
+		#[cfg(feature = "kvdb-rocksdb")]
+		DatabaseSettingsSrc::Path { path, cache_size } => {
+			let mut db_config = DatabaseConfig::with_columns(Some(NUM_COLUMNS));
+			db_config.memory_budget = *cache_size;
+			let path = path.to_str().ok_or_else(|| client::error::Error::Backend("Invalid database path".into()))?;
+			Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
+		},
+		#[cfg(not(feature = "kvdb-rocksdb"))]
+		DatabaseSettingsSrc::Path { .. } => {
+			let msg = "Try to open RocksDB database with RocksDB disabled".into();
+			return Err(client::error::Error::Backend(msg));
+		},
+		DatabaseSettingsSrc::Custom(db) => db.clone(),
+	};
+
+	// check database type
+	match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
+		Some(stored_type) => {
+			if db_type.as_bytes() != &*stored_type {
+				return Err(client::error::Error::Backend(
+					format!("Unexpected database type. Expected: {}", db_type)).into());
+			}
+		},
+		None => {
+			let mut transaction = DBTransaction::new();
+			transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
+			db.write(transaction).map_err(db_err)?;
+		},
+	}
+
+	Ok(db)
+}
+
+/// Read database column entry for the given block.
+pub fn read_db<Block>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>
+) -> client::error::Result<Option<DBValue>>
+	where
+		Block: BlockT,
+{
+	block_id_to_lookup_key(db, col_index, id).and_then(|key| match key {
+		Some(key) => db.get(col, key.as_ref()).map_err(db_err),
+		None => Ok(None),
+	})
+}
+
+/// Read a header from the database.
+pub fn read_header<Block: BlockT>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>,
+) -> client::error::Result<Option<Block::Header>> {
+	match read_db(db, col_index, col, id)? {
+		Some(header) => match Block::Header::decode(&mut &header[..]) {
+			Ok(header) => Ok(Some(header)),
+			Err(_) => return Err(
+				client::error::Error::Backend("Error decoding header".into())
+			),
+		}
+		None => Ok(None),
+	}
+}
+
+/// Required header from the database.
+pub fn require_header<Block: BlockT>(
+	db: &dyn KeyValueDB,
+	col_index: Option<u32>,
+	col: Option<u32>,
+	id: BlockId<Block>,
+) -> client::error::Result<Block::Header> {
+	read_header(db, col_index, col, id)
+		.and_then(|header| header.ok_or_else(|| client::error::Error::UnknownBlock(format!("{}", id))))
+}
+
+/// Read meta from the database.
+pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: Option<u32>, col_header: Option<u32>) -> Result<
+	Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
+	client::error::Error,
+>
+	where
+		Block: BlockT,
+{
+	let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
+		Some(h) => match Decode::decode(&mut &h[..]) {
+			Ok(h) => h,
+			Err(err) => return Err(client::error::Error::Backend(
+				format!("Error decoding genesis hash: {}", err)
+			)),
+		},
+		None => return Ok(Meta {
+			best_hash: Default::default(),
+			best_number: Zero::zero(),
+			finalized_hash: Default::default(),
+			finalized_number: Zero::zero(),
+			genesis_hash: Default::default(),
+		}),
+	};
+
+	let load_meta_block = |desc, key| -> Result<_, client::error::Error> {
+		if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
+			match id {
+				Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
+				None => Ok(None),
+			}).map_err(db_err)?
+		{
+			let hash = header.hash();
+			debug!("DB Opened blockchain db, fetched {} = {:?} ({})", desc, hash, header.number());
+			Ok((hash, *header.number()))
+		} else {
+			Ok((genesis_hash.clone(), Zero::zero()))
+		}
+	};
+
+	let (best_hash, best_number) = load_meta_block("best", meta_keys::BEST_BLOCK)?;
+	let (finalized_hash, finalized_number) = load_meta_block("final", meta_keys::FINALIZED_BLOCK)?;
+
+	Ok(Meta {
+		best_hash,
+		best_number,
+		finalized_hash,
+		finalized_number,
+		genesis_hash,
+	})
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use sr_primitives::testing::{Block as RawBlock, ExtrinsicWrapper};
+	type Block = RawBlock<ExtrinsicWrapper<u32>>;
+
+	#[test]
+	fn number_index_key_doesnt_panic() {
+		let id = BlockId::<Block>::Number(72340207214430721);
+		match id {
+			BlockId::Number(n) => number_index_key(n).expect_err("number should overflow u32"),
+			_ => unreachable!(),
+		};
+	}
+}