Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure * Remove old CI stuff * add title * formatting fixes * move node-exits job's script to scripts dir * Move docs into subdir * move to bin * move maintainence scripts, configs and helpers into its own dir * add .local to ignore * move core->client * start up 'test' area * move test client * move test runtime * make test move compile * Add dependencies rule enforcement. * Fix indexing. * Update docs to reflect latest changes * Moving /srml->/paint * update docs * move client/sr-* -> primitives/ * clean old readme * remove old broken code in rhd * update lock * Step 1. * starting to untangle client * Fix after merge. * start splitting out client interfaces * move children and blockchain interfaces * Move trie and state-machine to primitives. * Fix WASM builds. * fixing broken imports * more interface moves * move backend and light to interfaces * move CallExecutor * move cli off client * moving around more interfaces * re-add consensus crates into the mix * fix subkey path * relieve client from executor * starting to pull out client from grandpa * move is_decendent_of out of client * grandpa still depends on client directly * lemme tests pass * rename srml->paint * Make it compile. * rename interfaces->client-api * Move keyring to primitives. * fixup libp2p dep * fix broken use * allow dependency enforcement to fail * move fork-tree * Moving wasm-builder * make env * move build-script-utils * fixup broken crate depdencies and names * fix imports for authority discovery * fix typo * update cargo.lock * fixing imports * Fix paths and add missing crates * re-add missing crates
2026-06-12 10:01:17 +00:00 · 2019-11-14 21:51:17 +01:00
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
@@ -0,0 +1,161 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage entries.
+
+use client_api::error::Result as ClientResult;
+use sr_primitives::traits::{Block as BlockT, NumberFor};
+use codec::{Encode, Decode};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_storage::{Storage};
+
+/// Single list-based cache entry.
+#[derive(Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct Entry<Block: BlockT, T> {
+	/// first block, when this value became actual.
+	pub valid_from: ComplexBlockId<Block>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+/// Internal representation of the single list-based cache entry. The entry points to the
+/// previous entry in the cache, allowing us to traverse back in time in list-style.
+#[derive(Debug, Encode, Decode)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct StorageEntry<Block: BlockT, T: CacheItemT> {
+	/// None if valid from the beginning.
+	pub prev_valid_from: Option<ComplexBlockId<Block>>,
+	/// Value stored at this entry.
+	pub value: T,
+}
+
+impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
+	/// Returns Some if the entry should be updated with the new value.
+	pub fn try_update(&self, value: Option<T>) -> Option<StorageEntry<Block, T>> {
+		match value {
+			Some(value) => match self.value == value {
+				true => None,
+				false => Some(StorageEntry {
+					prev_valid_from: Some(self.valid_from.clone()),
+					value,
+				}),
+			},
+			None => None,
+		}
+	}
+
+	/// Wrapper that calls search_before to get range where the given block fits.
+	pub fn search_best_range_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
+		Ok(self.search_best_before(storage, block)?
+			.map(|(entry, next)| (entry.valid_from, next)))
+	}
+
+	/// Searches the list, ending with THIS entry for the best entry preceeding (or at)
+	/// given block number.
+	/// If the entry is found, result is the entry and the block id of next entry (if exists).
+	/// NOTE that this function does not check that the passed block is actually linked to
+	/// the blocks it found.
+	pub fn search_best_before<S: Storage<Block, T>>(
+		&self,
+		storage: &S,
+		block: NumberFor<Block>,
+	) -> ClientResult<Option<(Entry<Block, T>, Option<ComplexBlockId<Block>>)>> {
+		// we're looking for the best value
+		let mut next = None;
+		let mut current = self.valid_from.clone();
+		if block >= self.valid_from.number {
+			let value = self.value.clone();
+			return Ok(Some((Entry { valid_from: current, value }, next)));
+		}
+
+		// else - travel back in time
+		loop {
+			let entry = storage.require_entry(&current)?;
+			if block >= current.number {
+				return Ok(Some((Entry { valid_from: current, value: entry.value }, next)));
+			}
+
+			next = Some(current);
+			current = match entry.prev_valid_from {
+				Some(prev_valid_from) => prev_valid_from,
+				None => return Ok(None),
+			};
+		}
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
+	/// Converts storage entry into an entry, valid from given block.
+	pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
+		Entry {
+			valid_from,
+			value: self.value,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::cache::list_cache::tests::test_id;
+	use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
+	use super::*;
+
+	#[test]
+	fn entry_try_update_works() {
+		// when trying to update with None value
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(1), value: 42 }.try_update(None), None);
+		// when trying to update with the same Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
+		// when trying to update with different Some value
+		assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
+			Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }));
+	}
+
+	#[test]
+	fn entry_search_best_before_fails() {
+		// when storage returns error
+		assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
+			.search_best_before(&FaultyStorage, 50).is_err());
+	}
+
+	#[test]
+	fn entry_search_best_before_works() {
+		// when block is better than our best block
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new(), 150).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)));
+		// when block is found between two entries
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }),
+			75).unwrap(),
+		Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))));
+		// when block is not found
+		assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
+			.search_best_before(&DummyStorage::new()
+				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
+				.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
+			30).unwrap(),
+		None);
+	}
+}
@@ -0,0 +1,386 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! List-cache storage definition and implementation.
+
+use std::sync::Arc;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::error::{Error as ClientError, Result as ClientResult};
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+use crate::utils::{self, db_err, meta_keys};
+
+use crate::cache::{CacheItemT, ComplexBlockId};
+use crate::cache::list_cache::{CommitOperation, Fork};
+use crate::cache::list_entry::{Entry, StorageEntry};
+
+/// Single list-cache metadata.
+#[derive(Debug)]
+#[cfg_attr(test, derive(Clone, PartialEq))]
+pub struct Metadata<Block: BlockT> {
+	/// Block at which best finalized entry is stored.
+	pub finalized: Option<ComplexBlockId<Block>>,
+	/// A set of blocks at which best unfinalized entries are stored.
+	pub unfinalized: Vec<ComplexBlockId<Block>>,
+}
+
+/// Readonly list-cache storage trait.
+pub trait Storage<Block: BlockT, T: CacheItemT> {
+	/// Reads hash of the block at given number.
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>>;
+
+	/// Reads header of the block with given hash.
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>>;
+
+	/// Reads cache metadata: best finalized entry (if some) and the list.
+	fn read_meta(&self) -> ClientResult<Metadata<Block>>;
+
+	/// Reads cache entry from the storage.
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>>;
+
+	/// Reads referenced (and thus existing) cache entry from the storage.
+	fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
+		self.read_entry(at)
+			.and_then(|entry| entry
+				.ok_or_else(|| ClientError::from(
+					ClientError::Backend(format!("Referenced cache entry at {:?} is not found", at)))))
+	}
+}
+
+/// List-cache storage transaction.
+pub trait StorageTransaction<Block: BlockT, T: CacheItemT> {
+	/// Insert storage entry at given block.
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>);
+
+	/// Delete storage entry at given block.
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>);
+
+	/// Update metadata of the cache.
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	);
+}
+
+/// A set of columns used by the DbStorage.
+#[derive(Debug)]
+pub struct DbColumns {
+	/// Column holding cache meta.
+	pub meta: Option<u32>,
+	/// Column holding the mapping of { block number => block hash } for blocks of the best chain.
+	pub key_lookup: Option<u32>,
+	/// Column holding the mapping of { block hash => block header }.
+	pub header: Option<u32>,
+	/// Column holding cache entries.
+	pub cache: Option<u32>,
+}
+
+/// Database-backed list cache storage.
+pub struct DbStorage {
+	name: Vec<u8>,
+	meta_key: Vec<u8>,
+	db: Arc<dyn KeyValueDB>,
+	columns: DbColumns,
+}
+
+impl DbStorage {
+	/// Create new database-backed list cache storage.
+	pub fn new(name: Vec<u8>, db: Arc<dyn KeyValueDB>, columns: DbColumns) -> Self {
+		let meta_key = meta::key(&name);
+		DbStorage { name, meta_key, db, columns }
+	}
+
+	/// Get reference to the database.
+	pub fn db(&self) -> &Arc<dyn KeyValueDB> { &self.db }
+
+	/// Get reference to the database columns.
+	pub fn columns(&self) -> &DbColumns { &self.columns }
+
+	/// Encode block id for storing as a key in cache column.
+	/// We append prefix to the actual encoding to allow several caches
+	/// store entries in the same column.
+	pub fn encode_block_id<Block: BlockT>(&self, block: &ComplexBlockId<Block>) -> Vec<u8> {
+		let mut encoded = self.name.clone();
+		encoded.extend(block.hash.as_ref());
+		encoded
+	}
+}
+
+impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
+	fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Number(at))
+			.map(|maybe_header| maybe_header.map(|header| header.hash()))
+	}
+
+	fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+		utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Hash(*at))
+	}
+
+	fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+		self.db.get(self.columns.meta, &self.meta_key)
+			.map_err(db_err)
+			.and_then(|meta| match meta {
+				Some(meta) => meta::decode(&*meta),
+				None => Ok(Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				}),
+			})
+	}
+
+	fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+		self.db.get(self.columns.cache, &self.encode_block_id(at))
+			.map_err(db_err)
+			.and_then(|entry| match entry {
+				Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
+					.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
+					.map(Some),
+				None => Ok(None),
+			})
+	}
+}
+
+/// Database-backed list cache storage transaction.
+pub struct DbStorageTransaction<'a> {
+	storage: &'a DbStorage,
+	tx: &'a mut DBTransaction,
+}
+
+impl<'a> DbStorageTransaction<'a> {
+	/// Create new database transaction.
+	pub fn new(storage: &'a DbStorage, tx: &'a mut DBTransaction) -> Self {
+		DbStorageTransaction { storage, tx }
+	}
+}
+
+impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
+	fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
+		self.tx.put(self.storage.columns.cache, &self.storage.encode_block_id(at), &entry.encode());
+	}
+
+	fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+		self.tx.delete(self.storage.columns.cache, &self.storage.encode_block_id(at));
+	}
+
+	fn update_meta(
+		&mut self,
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		operation: &CommitOperation<Block, T>,
+	) {
+		self.tx.put(
+			self.storage.columns.meta,
+			&self.storage.meta_key,
+			&meta::encode(best_finalized_entry, unfinalized, operation));
+	}
+}
+
+/// Metadata related functions.
+mod meta {
+	use super::*;
+
+	/// Convert cache name into cache metadata key.
+	pub fn key(name: &[u8]) -> Vec<u8> {
+		let mut key_name = meta_keys::CACHE_META_PREFIX.to_vec();
+		key_name.extend_from_slice(name);
+		key_name
+	}
+
+	/// Encode cache metadata 'applying' commit operation before encoding.
+	pub fn encode<Block: BlockT, T: CacheItemT>(
+		best_finalized_entry: Option<&Entry<Block, T>>,
+		unfinalized: &[Fork<Block, T>],
+		op: &CommitOperation<Block, T>
+	) -> Vec<u8> {
+		let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
+		let mut unfinalized = unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
+
+		match op {
+			CommitOperation::AppendNewBlock(_, _) => (),
+			CommitOperation::AppendNewEntry(index, ref entry) => {
+				unfinalized[*index] = &entry.valid_from;
+			},
+			CommitOperation::AddNewFork(ref entry) => {
+				unfinalized.push(&entry.valid_from);
+			},
+			CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
+				finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
+				for fork_index in forks.iter().rev() {
+					unfinalized.remove(*fork_index);
+				}
+			},
+			CommitOperation::BlockReverted(ref forks) => {
+				for (fork_index, updated_fork) in forks.iter().rev() {
+					match updated_fork {
+						Some(updated_fork) => unfinalized[*fork_index] = &updated_fork.head().valid_from,
+						None => { unfinalized.remove(*fork_index); },
+					}
+				}
+			},
+		}
+
+		(finalized, unfinalized).encode()
+	}
+
+	/// Decode meta information.
+	pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
+		let input = &mut &*encoded;
+		let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+		let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input)
+			.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
+
+		Ok(Metadata { finalized, unfinalized })
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use std::collections::{HashMap, HashSet};
+	use super::*;
+
+	pub struct FaultyStorage;
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for FaultyStorage {
+		fn read_id(&self, _at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_header(&self, _at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+
+		fn read_entry(&self, _at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Err(ClientError::Backend("TestError".into()))
+		}
+	}
+
+	pub struct DummyStorage<Block: BlockT, T: CacheItemT> {
+		meta: Metadata<Block>,
+		ids: HashMap<NumberFor<Block>, Block::Hash>,
+		headers: HashMap<Block::Hash, Block::Header>,
+		entries: HashMap<Block::Hash, StorageEntry<Block, T>>,
+	}
+
+	impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
+		pub fn new() -> Self {
+			DummyStorage {
+				meta: Metadata {
+					finalized: None,
+					unfinalized: Vec::new(),
+				},
+				ids: HashMap::new(),
+				headers: HashMap::new(),
+				entries: HashMap::new(),
+			}
+		}
+
+		pub fn with_meta(mut self, finalized: Option<ComplexBlockId<Block>>, unfinalized: Vec<ComplexBlockId<Block>>) -> Self {
+			self.meta.finalized = finalized;
+			self.meta.unfinalized = unfinalized;
+			self
+		}
+
+		pub fn with_id(mut self, at: NumberFor<Block>, id: Block::Hash) -> Self {
+			self.ids.insert(at, id);
+			self
+		}
+
+		pub fn with_header(mut self, header: Block::Header) -> Self {
+			self.headers.insert(header.hash(), header);
+			self
+		}
+
+		pub fn with_entry(mut self, at: ComplexBlockId<Block>, entry: StorageEntry<Block, T>) -> Self {
+			self.entries.insert(at.hash, entry);
+			self
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DummyStorage<Block, T> {
+		fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
+			Ok(self.ids.get(&at).cloned())
+		}
+
+		fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
+			Ok(self.headers.get(&at).cloned())
+		}
+
+		fn read_meta(&self) -> ClientResult<Metadata<Block>> {
+			Ok(self.meta.clone())
+		}
+
+		fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
+			Ok(self.entries.get(&at.hash).cloned())
+		}
+	}
+
+	pub struct DummyTransaction<Block: BlockT> {
+		updated_meta: Option<Metadata<Block>>,
+		inserted_entries: HashSet<Block::Hash>,
+		removed_entries: HashSet<Block::Hash>,
+	}
+
+	impl<Block: BlockT> DummyTransaction<Block> {
+		pub fn new() -> Self {
+			DummyTransaction {
+				updated_meta: None,
+				inserted_entries: HashSet::new(),
+				removed_entries: HashSet::new(),
+			}
+		}
+
+		pub fn inserted_entries(&self) -> &HashSet<Block::Hash> {
+			&self.inserted_entries
+		}
+
+		pub fn removed_entries(&self) -> &HashSet<Block::Hash> {
+			&self.removed_entries
+		}
+
+		pub fn updated_meta(&self) -> &Option<Metadata<Block>> {
+			&self.updated_meta
+		}
+	}
+
+	impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
+		fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, _entry: &StorageEntry<Block, T>) {
+			self.inserted_entries.insert(at.hash);
+		}
+
+		fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
+			self.removed_entries.insert(at.hash);
+		}
+
+		fn update_meta(
+			&mut self,
+			best_finalized_entry: Option<&Entry<Block, T>>,
+			unfinalized: &[Fork<Block, T>],
+			operation: &CommitOperation<Block, T>,
+		) {
+			self.updated_meta = Some(meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap());
+		}
+	}
+}
@@ -0,0 +1,371 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! DB-backed cache of blockchain data.
+
+use std::{sync::Arc, collections::HashMap};
+use parking_lot::RwLock;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
+use client_api::error::Result as ClientResult;
+use codec::{Encode, Decode};
+use sr_primitives::generic::BlockId;
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
+use crate::utils::{self, COLUMN_META, db_err};
+
+use self::list_cache::{ListCache, PruningStrategy};
+
+mod list_cache;
+mod list_entry;
+mod list_storage;
+
+/// Minimal post-finalization age of finalized blocks before they'll pruned.
+const PRUNE_DEPTH: u32 = 1024;
+
+/// The type of entry that is inserted to the cache.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum EntryType {
+	/// Non-final entry.
+	NonFinal,
+	/// Final entry.
+	Final,
+	/// Genesis entry (inserted during cache initialization).
+	Genesis,
+}
+
+/// Block identifier that holds both hash and number.
+#[derive(Clone, Debug, Encode, Decode, PartialEq)]
+pub struct ComplexBlockId<Block: BlockT> {
+	hash: Block::Hash,
+	number: NumberFor<Block>,
+}
+
+impl<Block: BlockT> ComplexBlockId<Block> {
+	/// Create new complex block id.
+	pub fn new(hash: Block::Hash, number: NumberFor<Block>) -> Self {
+		ComplexBlockId { hash, number }
+	}
+}
+
+impl<Block: BlockT> ::std::cmp::PartialOrd for ComplexBlockId<Block> {
+	fn partial_cmp(&self, other: &ComplexBlockId<Block>) -> Option<::std::cmp::Ordering> {
+		self.number.partial_cmp(&other.number)
+	}
+}
+
+/// All cache items must implement this trait.
+pub trait CacheItemT: Clone + Decode + Encode + PartialEq {}
+
+impl<T> CacheItemT for T where T: Clone + Decode + Encode + PartialEq {}
+
+/// Database-backed blockchain data cache.
+pub struct DbCache<Block: BlockT> {
+	cache_at: HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	db: Arc<dyn KeyValueDB>,
+	key_lookup_column: Option<u32>,
+	header_column: Option<u32>,
+	authorities_column: Option<u32>,
+	genesis_hash: Block::Hash,
+	best_finalized_block: ComplexBlockId<Block>,
+}
+
+impl<Block: BlockT> DbCache<Block> {
+	/// Create new cache.
+	pub fn new(
+		db: Arc<dyn KeyValueDB>,
+		key_lookup_column: Option<u32>,
+		header_column: Option<u32>,
+		authorities_column: Option<u32>,
+		genesis_hash: Block::Hash,
+		best_finalized_block: ComplexBlockId<Block>,
+	) -> Self {
+		Self {
+			cache_at: HashMap::new(),
+			db,
+			key_lookup_column,
+			header_column,
+			authorities_column,
+			genesis_hash,
+			best_finalized_block,
+		}
+	}
+
+	/// Set genesis block hash.
+	pub fn set_genesis_hash(&mut self, genesis_hash: Block::Hash) {
+		self.genesis_hash = genesis_hash;
+	}
+
+	/// Begin cache transaction.
+	pub fn transaction<'a>(&'a mut self, tx: &'a mut DBTransaction) -> DbCacheTransaction<'a, Block> {
+		DbCacheTransaction {
+			cache: self,
+			tx,
+			cache_at_op: HashMap::new(),
+			best_finalized_block: None,
+		}
+	}
+
+	/// Run post-commit cache operations.
+	pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
+		for (name, op) in ops.cache_at_op.into_iter() {
+			self.get_cache(name).on_transaction_commit(op);
+		}
+		if let Some(best_finalized_block) = ops.best_finalized_block {
+			self.best_finalized_block = best_finalized_block;
+		}
+	}
+
+	/// Creates `ListCache` with the given name or returns a reference to the existing.
+	fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+		get_cache_helper(
+			&mut self.cache_at,
+			name,
+			&self.db,
+			self.key_lookup_column,
+			self.header_column,
+			self.authorities_column,
+			&self.best_finalized_block
+		)
+	}
+}
+
+// This helper is needed because otherwise the borrow checker will require to
+// clone all parameters outside of the closure.
+fn get_cache_helper<'a, Block: BlockT>(
+	cache_at: &'a mut HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
+	name: CacheKeyId,
+	db: &Arc<dyn KeyValueDB>,
+	key_lookup: Option<u32>,
+	header: Option<u32>,
+	cache: Option<u32>,
+	best_finalized_block: &ComplexBlockId<Block>,
+) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
+	cache_at.entry(name).or_insert_with(|| {
+		ListCache::new(
+			self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
+				self::list_storage::DbColumns {
+					meta: COLUMN_META,
+					key_lookup,
+					header,
+					cache,
+				},
+			),
+			cache_pruning_strategy(name),
+			best_finalized_block.clone(),
+		)
+	})
+}
+
+/// Cache operations that are to be committed after database transaction is committed.
+pub struct DbCacheTransactionOps<Block: BlockT> {
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+/// Database-backed blockchain data cache transaction valid for single block import.
+pub struct DbCacheTransaction<'a, Block: BlockT> {
+	cache: &'a mut DbCache<Block>,
+	tx: &'a mut DBTransaction,
+	cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
+	best_finalized_block: Option<ComplexBlockId<Block>>,
+}
+
+impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
+	/// Convert transaction into post-commit operations set.
+	pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
+		DbCacheTransactionOps {
+			cache_at_op: self.cache_at_op,
+			best_finalized_block: self.best_finalized_block,
+		}
+	}
+
+	/// When new block is inserted into database.
+	pub fn on_block_insert(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>,
+		data_at: HashMap<CacheKeyId, Vec<u8>>,
+		entry_type: EntryType,
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		// prepare list of caches that are not update
+		// (we might still need to do some cache maintenance in this case)
+		let missed_caches = self.cache.cache_at.keys()
+			.filter(|cache| !data_at.contains_key(*cache))
+			.cloned()
+			.collect::<Vec<_>>();
+
+		let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), client_api::error::Error> {
+			let cache = self.cache.get_cache(name);
+			let op = cache.on_block_insert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx,
+				),
+				parent.clone(),
+				block.clone(),
+				value,
+				entry_type,
+			)?;
+			if let Some(op) = op {
+				self.cache_at_op.insert(name, op);
+			}
+			Ok(())
+		};
+
+		data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?;
+		missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
+
+		match entry_type {
+			EntryType::Final | EntryType::Genesis =>
+				self.best_finalized_block = Some(block),
+			EntryType::NonFinal => (),
+		}
+
+		Ok(self)
+	}
+
+	/// When previously inserted block is finalized.
+	pub fn on_block_finalize(
+		mut self,
+		parent: ComplexBlockId<Block>,
+		block: ComplexBlockId<Block>
+	) -> ClientResult<Self> {
+		assert!(self.cache_at_op.is_empty());
+
+		for (name, cache_at) in self.cache.cache_at.iter() {
+			let op = cache_at.on_block_finalize(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache_at.storage(),
+					&mut self.tx
+				),
+				parent.clone(),
+				block.clone(),
+			)?;
+
+			if let Some(op) = op {
+				self.cache_at_op.insert(name.to_owned(), op);
+			}
+		}
+
+		self.best_finalized_block = Some(block);
+
+		Ok(self)
+	}
+
+	/// When block is reverted.
+	pub fn on_block_revert(
+		mut self,
+		reverted_block: &ComplexBlockId<Block>,
+	) -> ClientResult<Self> {
+		for (name, cache) in self.cache.cache_at.iter() {
+			let op = cache.on_block_revert(
+				&mut self::list_storage::DbStorageTransaction::new(
+					cache.storage(),
+					&mut self.tx
+				),
+				reverted_block,
+			)?;
+
+			assert!(!self.cache_at_op.contains_key(name));
+			self.cache_at_op.insert(name.to_owned(), op);
+		}
+
+		Ok(self)
+	}
+}
+
+/// Synchronous implementation of database-backed blockchain data cache.
+pub struct DbCacheSync<Block: BlockT>(pub RwLock<DbCache<Block>>);
+
+impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
+	fn initialize(&self, key: &CacheKeyId, data: Vec<u8>) -> ClientResult<()> {
+		let mut cache = self.0.write();
+		let genesis_hash = cache.genesis_hash;
+		let cache_contents = vec![(*key, data)].into_iter().collect();
+		let db = cache.db.clone();
+		let mut dbtx = DBTransaction::new();
+		let tx = cache.transaction(&mut dbtx);
+		let tx = tx.on_block_insert(
+			ComplexBlockId::new(Default::default(), Zero::zero()),
+			ComplexBlockId::new(genesis_hash, Zero::zero()),
+			cache_contents,
+			EntryType::Genesis,
+		)?;
+		let tx_ops = tx.into_ops();
+		db.write(dbtx).map_err(db_err)?;
+		cache.commit(tx_ops);
+		Ok(())
+	}
+
+	fn get_at(
+		&self,
+		key: &CacheKeyId,
+		at: &BlockId<Block>,
+	) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
+		let mut cache = self.0.write();
+		let storage = cache.get_cache(*key).storage();
+		let db = storage.db();
+		let columns = storage.columns();
+		let at = match *at {
+			BlockId::Hash(hash) => {
+				let header = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Hash(hash.clone())).ok()??;
+				ComplexBlockId::new(hash, *header.number())
+			},
+			BlockId::Number(number) => {
+				let hash = utils::read_header::<Block>(
+					&**db,
+					columns.key_lookup,
+					columns.header,
+					BlockId::Number(number.clone())).ok()??.hash();
+				ComplexBlockId::new(hash, number)
+			},
+		};
+
+		cache.cache_at
+			.get(key)?
+			.value_at_block(&at)
+			.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
+				(
+					(begin_block.number, begin_block.hash),
+					end_block.map(|end_block| (end_block.number, end_block.hash)),
+					value,
+				)))
+			.ok()?
+	}
+}
+
+/// Get pruning strategy for given cache.
+fn cache_pruning_strategy<N: From<u32>>(cache: CacheKeyId) -> PruningStrategy<N> {
+	// the cache is mostly used to store data from consensus engines
+	// this kind of data is only required for non-finalized blocks
+	// => by default we prune finalized cached entries
+
+	match cache {
+		// we need to keep changes tries configurations forever (or at least until changes tries,
+		// that were built using this configuration, are pruned) to make it possible to refer
+		// to old changes tries
+		well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune,
+		_ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()),
+	}
+}