Phase 1 of repo reorg (#719)

* Remove unneeded script

* Rename Substrate Demo -> Substrate

* Rename demo -> node

* Build wasm from last rename.

* Merge ed25519 into substrate-primitives

* Minor tweak

* Rename substrate -> core

* Move substrate-runtime-support to core/runtime/support

* Rename/move substrate-runtime-version

* Move codec up a level

* Rename substrate-codec -> parity-codec

* Move environmental up a level

* Move pwasm-* up to top, ready for removal

* Remove requirement of s-r-support from s-r-primitives

* Move core/runtime/primitives into core/runtime-primitives

* Remove s-r-support dep from s-r-version

* Remove dep of s-r-support from bft

* Remove dep of s-r-support from node/consensus

* Sever all other core deps from s-r-support

* Forgot the no_std directive

* Rename non-SRML modules to sr-* to avoid match clashes

* Move runtime/* to srml/*

* Rename substrate-runtime-* -> srml-*

* Move srml to top-level

substrate/core/client/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "substrate-client"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+error-chain = "0.12"
+fnv = "1.0"
+log = "0.3"
+parking_lot = "0.4"
+triehash = "0.2"
+hex-literal = "0.1"
+futures = "0.1.17"
+slog = "^2"
+heapsize = "0.4"
+substrate-bft = { path = "../bft" }
+parity-codec = { path = "../../codec" }
+substrate-executor = { path = "../executor" }
+substrate-primitives = { path = "../primitives" }
+sr-io = { path = "../sr-io" }
+substrate-metadata = { path = "../metadata" }
+sr-primitives = { path = "../sr-primitives" }
+substrate-state-machine = { path = "../state-machine" }
+substrate-keyring = { path = "../../core/keyring" }
+substrate-telemetry = { path = "../telemetry" }
+hashdb = "0.2.1"
+patricia-trie = "0.2.1"
+rlp = "0.2.4"
+
+[dev-dependencies]
+substrate-test-client = { path = "../test-client" }

substrate/core/client/README.adoc

@@ -0,0 +1,13 @@
+
+= Client
+
+.Summary
+[source, toml]
+----
+include::Cargo.toml[lines=2..5]
+----
+
+.Description
+----
+include::src/lib.rs[tag=description]
+----

substrate/core/client/db/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "substrate-client-db"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[dependencies]
+parking_lot = "0.4"
+log = "0.3"
+kvdb = "0.1"
+kvdb-rocksdb = "0.1.3"
+hashdb = "0.2.1"
+memorydb = "0.2.1"
+substrate-primitives = { path = "../../../core/primitives" }
+sr-primitives = { path = "../../../core/sr-primitives" }
+substrate-client = { path = "../../../core/client" }
+substrate-state-machine = { path = "../../../core/state-machine" }
+parity-codec = { path = "../../../codec" }
+parity-codec-derive = { path = "../../../codec/derive" }
+substrate-executor = { path = "../../../core/executor" }
+substrate-state-db = { path = "../../../core/state-db" }
+
+[dev-dependencies]
+kvdb-memorydb = "0.1"

substrate/core/client/db/src/cache.rs

@@ -0,0 +1,432 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! DB-backed cache of blockchain data.
+
+use std::sync::Arc;
+use parking_lot::RwLock;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client::blockchain::Cache as BlockchainCache;
+use client::error::Result as ClientResult;
+use codec::{Codec, Encode, Decode};
+use primitives::AuthorityId;
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{Block as BlockT, As, NumberFor};
+use utils::{COLUMN_META, BlockKey, db_err, meta_keys, read_id, db_key_to_number, number_to_db_key};
+
+/// Database-backed cache of blockchain data.
+pub struct DbCache<Block: BlockT> {
+	db: Arc<KeyValueDB>,
+	block_index_column: Option<u32>,
+	authorities_at: DbCacheList<Block, Vec<AuthorityId>>,
+}
+
+impl<Block> DbCache<Block>
+	where
+		Block: BlockT,
+		NumberFor<Block>: As<u64>,
+{
+	/// Create new cache.
+	pub fn new(
+		db: Arc<KeyValueDB>,
+		block_index_column: Option<u32>,
+		authorities_column: Option<u32>
+	) -> ClientResult<Self> {
+		Ok(DbCache {
+			db: db.clone(),
+			block_index_column,
+			authorities_at: DbCacheList::new(db, meta_keys::BEST_AUTHORITIES, authorities_column)?,
+		})
+	}
+
+	/// Get authorities_cache.
+	pub fn authorities_at_cache(&self) -> &DbCacheList<Block, Vec<AuthorityId>> {
+		&self.authorities_at
+	}
+}
+
+impl<Block> BlockchainCache<Block> for DbCache<Block>
+	where
+		Block: BlockT,
+		NumberFor<Block>: As<u64>,
+{
+	fn authorities_at(&self, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
+		let authorities_at = read_id(&*self.db, self.block_index_column, at).and_then(|at| match at {
+			Some(at) => self.authorities_at.value_at_key(at),
+			None => Ok(None),
+		});
+
+		match authorities_at {
+			Ok(authorities) => authorities,
+			Err(error) => {
+				warn!("Trying to read authorities from db cache has failed with: {}", error);
+				None
+			},
+		}
+	}
+}
+
+/// Database-backed blockchain cache which holds its entries as a list.
+/// The meta column holds the pointer to the best known cache entry and
+/// every entry points to the previous entry.
+/// New entry appears when the set of authorities changes in block, so the
+/// best entry here means the entry that is valid for the best block (and
+/// probably for its ascendants).
+pub struct DbCacheList<Block: BlockT, T: Clone> {
+	db: Arc<KeyValueDB>,
+	meta_key: &'static [u8],
+	column: Option<u32>,
+	/// Best entry at the moment. None means that cache has no entries at all.
+	best_entry: RwLock<Option<Entry<NumberFor<Block>, T>>>,
+}
+
+/// Single cache entry.
+#[derive(Clone)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub struct Entry<N, T: Clone> {
+	/// first block, when this value became actual
+	valid_from: N,
+	/// None means that we do not know the value starting from `valid_from` block
+	value: Option<T>,
+}
+
+/// Internal representation of the single cache entry. The entry points to the
+/// previous entry in the cache, allowing us to traverse back in time in list-style.
+#[derive(Encode, Decode)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+struct StorageEntry<N, T> {
+	/// None if valid from the beginning
+	prev_valid_from: Option<N>,
+	/// None means that we do not know the value starting from `valid_from` block
+	value: Option<T>,
+}
+
+impl<Block, T> DbCacheList<Block, T>
+	where
+		Block: BlockT,
+		NumberFor<Block>: As<u64>,
+		T: Clone + PartialEq + Codec,
+{
+	/// Creates new cache list.
+	fn new(db: Arc<KeyValueDB>, meta_key: &'static [u8], column: Option<u32>) -> ClientResult<Self> {
+		let best_entry = RwLock::new(db.get(COLUMN_META, meta_key)
+			.map_err(db_err)
+			.and_then(|block| match block {
+				Some(block) => {
+					let valid_from = db_key_to_number(&block)?;
+					read_storage_entry::<Block, T>(&*db, column, valid_from)
+						.map(|entry| Some(Entry {
+							valid_from,
+							value: entry
+								.expect("meta entry references the entry at the block; storage entry at block exists when referenced; qed")
+								.value,
+						}))
+				},
+				None => Ok(None),
+			})?);
+
+		Ok(DbCacheList {
+			db,
+			column,
+			meta_key,
+			best_entry,
+		})
+	}
+
+	/// Gets the best known entry.
+	pub fn best_entry(&self) -> Option<Entry<NumberFor<Block>, T>> {
+		self.best_entry.read().clone()
+	}
+
+	/// Commits the new best pending value to the database. Returns Some if best entry must
+	/// be updated after transaction is committed.
+	pub fn commit_best_entry(
+		&self,
+		transaction: &mut DBTransaction,
+		valid_from: NumberFor<Block>,
+		pending_value: Option<T>
+	) -> Option<Entry<NumberFor<Block>, T>> {
+		let best_entry = self.best_entry();
+		let update_best_entry = match (
+			best_entry.as_ref().and_then(|a| a.value.as_ref()),
+			pending_value.as_ref()
+		) {
+			(Some(best_value), Some(pending_value)) => best_value != pending_value,
+			(None, Some(_)) | (Some(_), None) => true,
+			(None, None) => false,
+		};
+		if !update_best_entry {
+			return None;
+		}
+
+		let valid_from_key = number_to_db_key(valid_from);
+		transaction.put(COLUMN_META, self.meta_key, &valid_from_key);
+		transaction.put(self.column, &valid_from_key, &StorageEntry {
+			prev_valid_from: best_entry.map(|b| b.valid_from),
+			value: pending_value.clone(),
+		}.encode());
+
+		Some(Entry {
+			valid_from,
+			value: pending_value,
+		})
+	}
+
+	/// Updates the best in-memory cache entry. Must be called after transaction with changes
+	/// from commit_best_entry has been committed.
+	pub fn update_best_entry(&self, best_entry: Option<Entry<NumberFor<Block>, T>>) {
+		*self.best_entry.write() = best_entry;
+	}
+
+	/// Prune all entries from the beginning up to the block (including entry at the number). Returns
+	/// the number of pruned entries. Pruning never deletes the latest entry in the cache.
+	pub fn prune_entries(
+		&self,
+		transaction: &mut DBTransaction,
+		last_to_prune: NumberFor<Block>
+	) -> ClientResult<usize> {
+		// find the last entry we want to keep
+		let mut last_entry_to_keep = match self.best_entry() {
+			Some(best_entry) => best_entry.valid_from,
+			None => return Ok(0),
+		};
+		let mut first_entry_to_remove = last_entry_to_keep;
+		while first_entry_to_remove > last_to_prune {
+			last_entry_to_keep = first_entry_to_remove;
+
+			let entry = read_storage_entry::<Block, T>(&*self.db, self.column, first_entry_to_remove)?
+				.expect("entry referenced from the next entry; entry exists when referenced; qed");
+			// if we have reached the first list entry
+			// AND all list entries are for blocks that are later than last_to_prune
+			// => nothing to prune
+			first_entry_to_remove = match entry.prev_valid_from {
+				Some(prev_valid_from) => prev_valid_from,
+				None => return Ok(0),
+			}
+		}
+
+		// remove all entries, starting from entry_to_remove
+		let mut pruned = 0;
+		let mut entry_to_remove = Some(first_entry_to_remove);
+		while let Some(current_entry) = entry_to_remove {
+			let entry = read_storage_entry::<Block, T>(&*self.db, self.column, current_entry)?
+				.expect("referenced entry exists; entry_to_remove is a reference to the entry; qed");
+
+			if current_entry != last_entry_to_keep {
+				transaction.delete(self.column, &number_to_db_key(current_entry));
+				pruned += 1;
+			}
+			entry_to_remove = entry.prev_valid_from;
+		}
+
+		let mut entry = read_storage_entry::<Block, T>(&*self.db, self.column, last_entry_to_keep)?
+			.expect("last_entry_to_keep >= first_entry_to_remove; that means that we're leaving this entry in the db; qed");
+		entry.prev_valid_from = None;
+		transaction.put(self.column, &number_to_db_key(last_entry_to_keep), &entry.encode());
+
+		Ok(pruned)
+	}
+
+	/// Reads the cached value, actual at given block. Returns None if the value was not cached
+	/// or if it has been pruned.
+	fn value_at_key(&self, key: BlockKey) -> ClientResult<Option<T>> {
+		let at = db_key_to_number::<NumberFor<Block>>(&key)?;
+		let best_valid_from = match self.best_entry() {
+			// there are entries in cache
+			Some(best_entry) => {
+				// we're looking for the best value
+				if at >= best_entry.valid_from {
+					return Ok(best_entry.value);
+				}
+
+				// we're looking for the value of older blocks
+				best_entry.valid_from
+			},
+			// there are no entries in the cache
+			None => return Ok(None),
+		};
+
+		let mut entry = read_storage_entry::<Block, T>(&*self.db, self.column, best_valid_from)?
+			.expect("self.best_entry().is_some() if there's entry for best_valid_from; qed");
+		loop {
+			let prev_valid_from = match entry.prev_valid_from {
+				Some(prev_valid_from) => prev_valid_from,
+				None => return Ok(None),
+			};
+
+			let prev_entry = read_storage_entry::<Block, T>(&*self.db, self.column, prev_valid_from)?
+				.expect("entry referenced from the next entry; entry exists when referenced; qed");
+			if at >= prev_valid_from {
+				return Ok(prev_entry.value);
+			}
+
+			entry = prev_entry;
+		}
+	}
+}
+
+/// Reads the entry at the block with given number.
+fn read_storage_entry<Block, T>(
+	db: &KeyValueDB,
+	column: Option<u32>,
+	number: NumberFor<Block>
+) -> ClientResult<Option<StorageEntry<NumberFor<Block>, T>>>
+	where
+		Block: BlockT,
+		NumberFor<Block>: As<u64>,
+		T: Codec,
+{
+	db.get(column, &number_to_db_key(number))
+		.and_then(|entry| match entry {
+			Some(entry) => Ok(StorageEntry::<NumberFor<Block>, T>::decode(&mut &entry[..])),
+			None => Ok(None),
+		})
+	.map_err(db_err)
+}
+
+#[cfg(test)]
+mod tests {
+	use runtime_primitives::testing::Block as RawBlock;
+	use light::{AUTHORITIES_ENTRIES_TO_KEEP, columns, LightStorage};
+	use light::tests::insert_block;
+	use super::*;
+
+	type Block = RawBlock<u64>;
+
+	#[test]
+	fn authorities_storage_entry_serialized() {
+		let test_cases: Vec<StorageEntry<u64, Vec<AuthorityId>>> = vec![
+			StorageEntry { prev_valid_from: Some(42), value: Some(vec![[1u8; 32].into()]) },
+			StorageEntry { prev_valid_from: None, value: Some(vec![[1u8; 32].into(), [2u8; 32].into()]) },
+			StorageEntry { prev_valid_from: None, value: None },
+		];
+
+		for expected in test_cases {
+			let serialized = expected.encode();
+			let deserialized = StorageEntry::decode(&mut &serialized[..]).unwrap();
+			assert_eq!(expected, deserialized);
+		}
+	}
+
+	#[test]
+	fn best_authorities_are_updated() {
+		let db = LightStorage::new_test();
+		let authorities_at: Vec<(usize, Option<Entry<u64, Vec<AuthorityId>>>)> = vec![
+			(0, None),
+			(0, None),
+			(1, Some(Entry { valid_from: 1, value: Some(vec![[2u8; 32].into()]) })),
+			(1, Some(Entry { valid_from: 1, value: Some(vec![[2u8; 32].into()]) })),
+			(2, Some(Entry { valid_from: 3, value: Some(vec![[4u8; 32].into()]) })),
+			(2, Some(Entry { valid_from: 3, value: Some(vec![[4u8; 32].into()]) })),
+			(3, Some(Entry { valid_from: 5, value: None })),
+			(3, Some(Entry { valid_from: 5, value: None })),
+		];
+
+		// before any block, there are no entries in cache
+		assert!(db.cache().authorities_at_cache().best_entry().is_none());
+		assert_eq!(db.db().iter(columns::AUTHORITIES).count(), 0);
+
+		// insert blocks and check that best_authorities() returns correct result
+		let mut prev_hash = Default::default();
+		for number in 0..authorities_at.len() {
+			let authorities_at_number = authorities_at[number].1.clone().and_then(|e| e.value);
+			prev_hash = insert_block(&db, &prev_hash, number as u64, authorities_at_number);
+			assert_eq!(db.cache().authorities_at_cache().best_entry(), authorities_at[number].1);
+			assert_eq!(db.db().iter(columns::AUTHORITIES).count(), authorities_at[number].0);
+		}
+
+		// check that authorities_at() returns correct results for all retrospective blocks
+		for number in 1..authorities_at.len() + 1 {
+			assert_eq!(db.cache().authorities_at(BlockId::Number(number as u64)),
+				authorities_at.get(number + 1)
+					.or_else(|| authorities_at.last())
+					.unwrap().1.clone().and_then(|e| e.value));
+		}
+
+		// now check that cache entries are pruned when new blocks are inserted
+		let mut current_entries_count = authorities_at.last().unwrap().0;
+		let pruning_starts_at = AUTHORITIES_ENTRIES_TO_KEEP as usize;
+		for number in authorities_at.len()..authorities_at.len() + pruning_starts_at {
+			prev_hash = insert_block(&db, &prev_hash, number as u64, None);
+			if number > pruning_starts_at {
+				let prev_entries_count = authorities_at[number - pruning_starts_at].0;
+				let entries_count = authorities_at.get(number - pruning_starts_at + 1).map(|e| e.0)
+					.unwrap_or_else(|| authorities_at.last().unwrap().0);
+				current_entries_count -= entries_count - prev_entries_count;
+			}
+
+			// there's always at least 1 entry in the cache (after first insertion)
+			assert_eq!(db.db().iter(columns::AUTHORITIES).count(), ::std::cmp::max(current_entries_count, 1));
+		}
+	}
+
+	#[test]
+	fn best_authorities_are_pruned() {
+		let db = LightStorage::<Block>::new_test();
+		let mut transaction = DBTransaction::new();
+
+		// insert first entry at block#100
+		db.cache().authorities_at_cache().update_best_entry(
+			db.cache().authorities_at_cache().commit_best_entry(&mut transaction, 100, Some(vec![[1u8; 32].into()])));
+		db.db().write(transaction).unwrap();
+
+		// no entries are pruned, since there's only one entry in the cache
+		let mut transaction = DBTransaction::new();
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 50).unwrap(), 0);
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 100).unwrap(), 0);
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 0);
+
+		// insert second entry at block#200
+		let mut transaction = DBTransaction::new();
+		db.cache().authorities_at_cache().update_best_entry(
+			db.cache().authorities_at_cache().commit_best_entry(&mut transaction, 200, Some(vec![[2u8; 32].into()])));
+		db.db().write(transaction).unwrap();
+
+		let mut transaction = DBTransaction::new();
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 50).unwrap(), 0);
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 100).unwrap(), 1);
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 1);
+		// still only 1 entry is removed since pruning never deletes the last entry
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 200).unwrap(), 1);
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 250).unwrap(), 1);
+
+		// physically remove entry for block#100 from db
+		let mut transaction = DBTransaction::new();
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 150).unwrap(), 1);
+		db.db().write(transaction).unwrap();
+
+		assert_eq!(db.cache().authorities_at_cache().best_entry().unwrap().value, Some(vec![[2u8; 32].into()]));
+		assert_eq!(db.cache().authorities_at(BlockId::Number(50)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(100)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(150)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(200)), Some(vec![[2u8; 32].into()]));
+		assert_eq!(db.cache().authorities_at(BlockId::Number(250)), Some(vec![[2u8; 32].into()]));
+
+		// try to delete last entry => failure (no entries are removed)
+		let mut transaction = DBTransaction::new();
+		assert_eq!(db.cache().authorities_at_cache().prune_entries(&mut transaction, 300).unwrap(), 0);
+		db.db().write(transaction).unwrap();
+
+		assert_eq!(db.cache().authorities_at_cache().best_entry().unwrap().value, Some(vec![[2u8; 32].into()]));
+		assert_eq!(db.cache().authorities_at(BlockId::Number(50)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(100)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(150)), None);
+		assert_eq!(db.cache().authorities_at(BlockId::Number(200)), Some(vec![[2u8; 32].into()]));
+		assert_eq!(db.cache().authorities_at(BlockId::Number(250)), Some(vec![[2u8; 32].into()]));
+	}
+}

substrate/core/client/db/src/lib.rs

@@ -0,0 +1,714 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Client backend that uses RocksDB database as storage.
+// end::description[]
+
+extern crate substrate_client as client;
+extern crate kvdb_rocksdb;
+extern crate kvdb;
+extern crate hashdb;
+extern crate memorydb;
+extern crate parking_lot;
+extern crate substrate_state_machine as state_machine;
+extern crate substrate_primitives as primitives;
+extern crate sr_primitives as runtime_primitives;
+extern crate parity_codec as codec;
+extern crate substrate_executor as executor;
+extern crate substrate_state_db as state_db;
+
+#[macro_use]
+extern crate log;
+
+#[macro_use]
+extern crate parity_codec_derive;
+
+#[cfg(test)]
+extern crate kvdb_memorydb;
+
+pub mod light;
+
+mod cache;
+mod utils;
+
+use std::sync::Arc;
+use std::path::PathBuf;
+use std::io;
+
+use codec::{Decode, Encode};
+use hashdb::Hasher;
+use kvdb::{KeyValueDB, DBTransaction};
+use memorydb::MemoryDB;
+use parking_lot::RwLock;
+use primitives::{H256, AuthorityId, Blake2Hasher, RlpCodec};
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::bft::Justification;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hash, HashFor, NumberFor, Zero};
+use runtime_primitives::BuildStorage;
+use state_machine::backend::Backend as StateBackend;
+use executor::RuntimeInfo;
+use state_machine::{CodeExecutor, DBValue, ExecutionStrategy};
+use utils::{Meta, db_err, meta_keys, number_to_db_key, db_key_to_number, open_database,
+	read_db, read_id, read_meta};
+use state_db::StateDb;
+pub use state_db::PruningMode;
+
+const FINALIZATION_WINDOW: u64 = 32;
+
+/// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
+pub type DbState = state_machine::TrieBackend<Blake2Hasher, RlpCodec>;
+
+/// Database settings.
+pub struct DatabaseSettings {
+	/// Cache size in bytes. If `None` default is used.
+	pub cache_size: Option<usize>,
+	/// Path to the database.
+	pub path: PathBuf,
+	/// Pruning mode.
+	pub pruning: PruningMode,
+}
+
+/// Create an instance of db-backed client.
+pub fn new_client<E, S, Block>(
+	settings: DatabaseSettings,
+	executor: E,
+	genesis_storage: S,
+	execution_strategy: ExecutionStrategy,
+) -> Result<client::Client<Backend<Block>, client::LocalCallExecutor<Backend<Block>, E>, Block>, client::error::Error>
+	where
+		Block: BlockT,
+		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+		S: BuildStorage,
+{
+	let backend = Arc::new(Backend::new(settings, FINALIZATION_WINDOW)?);
+	let executor = client::LocalCallExecutor::new(backend.clone(), executor);
+	Ok(client::Client::new(backend, executor, genesis_storage, execution_strategy)?)
+}
+
+mod columns {
+	pub const META: Option<u32> = Some(0);
+	pub const STATE: Option<u32> = Some(1);
+	pub const STATE_META: Option<u32> = Some(2);
+	pub const BLOCK_INDEX: Option<u32> = Some(3);
+	pub const HEADER: Option<u32> = Some(4);
+	pub const BODY: Option<u32> = Some(5);
+	pub const JUSTIFICATION: Option<u32> = Some(6);
+}
+
+struct PendingBlock<Block: BlockT> {
+	header: Block::Header,
+	justification: Option<Justification<Block::Hash>>,
+	body: Option<Vec<Block::Extrinsic>>,
+	is_best: bool,
+}
+
+// wrapper that implements trait required for state_db
+struct StateMetaDb<'a>(&'a KeyValueDB);
+
+impl<'a> state_db::MetaDb for StateMetaDb<'a> {
+	type Error = io::Error;
+
+	fn get_meta(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		self.0.get(columns::STATE_META, key).map(|r| r.map(|v| v.to_vec()))
+	}
+}
+
+/// Block database
+pub struct BlockchainDb<Block: BlockT> {
+	db: Arc<KeyValueDB>,
+	meta: RwLock<Meta<<Block::Header as HeaderT>::Number, Block::Hash>>,
+}
+
+impl<Block: BlockT> BlockchainDb<Block> {
+	fn new(db: Arc<KeyValueDB>) -> Result<Self, client::error::Error> {
+		let meta = read_meta::<Block>(&*db, columns::HEADER)?;
+		Ok(BlockchainDb {
+			db,
+			meta: RwLock::new(meta)
+		})
+	}
+
+	fn update_meta(&self, hash: Block::Hash, number: <Block::Header as HeaderT>::Number, is_best: bool) {
+		if is_best {
+			let mut meta = self.meta.write();
+			if number == Zero::zero() {
+				meta.genesis_hash = hash;
+			}
+			meta.best_number = number;
+			meta.best_hash = hash;
+		}
+	}
+}
+
+impl<Block: BlockT> client::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
+	fn header(&self, id: BlockId<Block>) -> Result<Option<Block::Header>, client::error::Error> {
+		match read_db(&*self.db, columns::BLOCK_INDEX, columns::HEADER, id)? {
+			Some(header) => match Block::Header::decode(&mut &header[..]) {
+				Some(header) => Ok(Some(header)),
+				None => return Err(client::error::ErrorKind::Backend("Error decoding header".into()).into()),
+			}
+			None => Ok(None),
+		}
+	}
+
+	fn info(&self) -> Result<client::blockchain::Info<Block>, client::error::Error> {
+		let meta = self.meta.read();
+		Ok(client::blockchain::Info {
+			best_hash: meta.best_hash,
+			best_number: meta.best_number,
+			genesis_hash: meta.genesis_hash,
+		})
+	}
+
+	fn status(&self, id: BlockId<Block>) -> Result<client::blockchain::BlockStatus, client::error::Error> {
+		let exists = match id {
+			BlockId::Hash(_) => read_id(&*self.db, columns::BLOCK_INDEX, id)?.is_some(),
+			BlockId::Number(n) => n <= self.meta.read().best_number,
+		};
+		match exists {
+			true => Ok(client::blockchain::BlockStatus::InChain),
+			false => Ok(client::blockchain::BlockStatus::Unknown),
+		}
+	}
+
+	fn number(&self, hash: Block::Hash) -> Result<Option<<Block::Header as HeaderT>::Number>, client::error::Error> {
+		read_id::<Block>(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash))
+			.and_then(|key| match key {
+				Some(key) => Ok(Some(db_key_to_number(&key)?)),
+				None => Ok(None),
+			})
+	}
+
+	fn hash(&self, number: <Block::Header as HeaderT>::Number) -> Result<Option<Block::Hash>, client::error::Error> {
+		read_db::<Block>(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x|
+			x.map(|raw| HashFor::<Block>::hash(&raw[..])).map(Into::into)
+		)
+	}
+}
+
+impl<Block: BlockT> client::blockchain::Backend<Block> for BlockchainDb<Block> {
+	fn body(&self, id: BlockId<Block>) -> Result<Option<Vec<Block::Extrinsic>>, client::error::Error> {
+		match read_db(&*self.db, columns::BLOCK_INDEX, columns::BODY, id)? {
+			Some(body) => match Decode::decode(&mut &body[..]) {
+				Some(body) => Ok(Some(body)),
+				None => return Err(client::error::ErrorKind::Backend("Error decoding body".into()).into()),
+			}
+			None => Ok(None),
+		}
+	}
+
+	fn justification(&self, id: BlockId<Block>) -> Result<Option<Justification<Block::Hash>>, client::error::Error> {
+		match read_db(&*self.db, columns::BLOCK_INDEX, columns::JUSTIFICATION, id)? {
+			Some(justification) => match Decode::decode(&mut &justification[..]) {
+				Some(justification) => Ok(Some(justification)),
+				None => return Err(client::error::ErrorKind::Backend("Error decoding justification".into()).into()),
+			}
+			None => Ok(None),
+		}
+	}
+
+	fn cache(&self) -> Option<&client::blockchain::Cache<Block>> {
+		None
+	}
+}
+
+/// Database transaction
+pub struct BlockImportOperation<Block: BlockT, H: Hasher> {
+	old_state: DbState,
+	updates: MemoryDB<H>,
+	pending_block: Option<PendingBlock<Block>>,
+}
+
+impl<Block> client::backend::BlockImportOperation<Block, Blake2Hasher, RlpCodec>
+for BlockImportOperation<Block, Blake2Hasher>
+where Block: BlockT,
+{
+	type State = DbState;
+
+	fn state(&self) -> Result<Option<&Self::State>, client::error::Error> {
+		Ok(Some(&self.old_state))
+	}
+
+	fn set_block_data(&mut self, header: Block::Header, body: Option<Vec<Block::Extrinsic>>, justification: Option<Justification<Block::Hash>>, is_best: bool) -> Result<(), client::error::Error> {
+		assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
+		self.pending_block = Some(PendingBlock {
+			header,
+			body,
+			justification,
+			is_best,
+		});
+		Ok(())
+	}
+
+	fn update_authorities(&mut self, _authorities: Vec<AuthorityId>) {
+		// currently authorities are not cached on full nodes
+	}
+
+	fn update_storage(&mut self, update: MemoryDB<Blake2Hasher>) -> Result<(), client::error::Error> {
+		self.updates = update;
+		Ok(())
+	}
+
+	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> Result<(), client::error::Error> {
+		// TODO: wipe out existing trie.
+		let (_, update) = self.old_state.storage_root(iter.into_iter().map(|(k, v)| (k, Some(v))));
+		self.updates = update;
+		Ok(())
+	}
+}
+
+struct StorageDb<Block: BlockT> {
+	pub db: Arc<KeyValueDB>,
+	pub state_db: StateDb<Block::Hash, H256>,
+}
+
+impl<Block: BlockT> state_machine::Storage<Blake2Hasher> for StorageDb<Block> {
+	fn get(&self, key: &H256) -> Result<Option<DBValue>, String> {
+		self.state_db.get(&key.0.into(), self).map(|r| r.map(|v| DBValue::from_slice(&v)))
+			.map_err(|e| format!("Database backend error: {:?}", e))
+	}
+}
+
+impl<Block: BlockT> state_db::HashDb for StorageDb<Block> {
+	type Error = io::Error;
+	type Hash = H256;
+
+	fn get(&self, key: &H256) -> Result<Option<Vec<u8>>, Self::Error> {
+		self.db.get(columns::STATE, &key[..]).map(|r| r.map(|v| v.to_vec()))
+	}
+}
+
+
+/// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
+/// Otherwise, trie nodes are kept only from the most recent block.
+pub struct Backend<Block: BlockT> {
+	storage: Arc<StorageDb<Block>>,
+	blockchain: BlockchainDb<Block>,
+	finalization_window: u64,
+}
+
+impl<Block: BlockT> Backend<Block> {
+	/// Create a new instance of database backend.
+	pub fn new(config: DatabaseSettings, finalization_window: u64) -> Result<Self, client::error::Error> {
+		let db = open_database(&config, "full")?;
+
+		Backend::from_kvdb(db as Arc<_>, config.pruning, finalization_window)
+	}
+
+	#[cfg(test)]
+	fn new_test(keep_blocks: u32) -> Self {
+		use utils::NUM_COLUMNS;
+
+		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
+
+		Backend::from_kvdb(db as Arc<_>, PruningMode::keep_blocks(keep_blocks), 0).expect("failed to create test-db")
+	}
+
+	fn from_kvdb(db: Arc<KeyValueDB>, pruning: PruningMode, finalization_window: u64) -> Result<Self, client::error::Error> {
+		let blockchain = BlockchainDb::new(db.clone())?;
+		let map_e = |e: state_db::Error<io::Error>| ::client::error::Error::from(format!("State database error: {:?}", e));
+		let state_db: StateDb<Block::Hash, H256> = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?;
+		let storage_db = StorageDb {
+			db,
+			state_db,
+		};
+
+		Ok(Backend {
+			storage: Arc::new(storage_db),
+			blockchain,
+			finalization_window,
+		})
+	}
+}
+
+fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitSet<H256>) {
+	for (key, val) in commit.data.inserted.into_iter() {
+		transaction.put(columns::STATE, &key[..], &val);
+	}
+	for key in commit.data.deleted.into_iter() {
+		transaction.delete(columns::STATE, &key[..]);
+	}
+	for (key, val) in commit.meta.inserted.into_iter() {
+		transaction.put(columns::STATE_META, &key[..], &val);
+	}
+	for key in commit.meta.deleted.into_iter() {
+		transaction.delete(columns::STATE_META, &key[..]);
+	}
+}
+
+impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<Block> where Block: BlockT {
+	type BlockImportOperation = BlockImportOperation<Block, Blake2Hasher>;
+	type Blockchain = BlockchainDb<Block>;
+	type State = DbState;
+
+	fn begin_operation(&self, block: BlockId<Block>) -> Result<Self::BlockImportOperation, client::error::Error> {
+		let state = self.state_at(block)?;
+		Ok(BlockImportOperation {
+			pending_block: None,
+			old_state: state,
+			updates: MemoryDB::default(),
+		})
+	}
+
+	fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> Result<(), client::error::Error> {
+		use client::blockchain::HeaderBackend;
+		let mut transaction = DBTransaction::new();
+		if let Some(pending_block) = operation.pending_block {
+			let hash = pending_block.header.hash();
+			let number = pending_block.header.number().clone();
+			let key = number_to_db_key(number.clone());
+			transaction.put(columns::HEADER, &key, &pending_block.header.encode());
+			if let Some(body) = pending_block.body {
+				transaction.put(columns::BODY, &key, &body.encode());
+			}
+			if let Some(justification) = pending_block.justification {
+				transaction.put(columns::JUSTIFICATION, &key, &justification.encode());
+			}
+			transaction.put(columns::BLOCK_INDEX, hash.as_ref(), &key);
+			if pending_block.is_best {
+				transaction.put(columns::META, meta_keys::BEST_BLOCK, &key);
+			}
+			let mut changeset: state_db::ChangeSet<H256> = state_db::ChangeSet::default();
+			for (key, (val, rc)) in operation.updates.drain() {
+				if rc > 0 {
+					changeset.inserted.push((key.0.into(), val.to_vec()));
+				} else if rc < 0 {
+					changeset.deleted.push(key.0.into());
+				}
+			}
+			let number_u64 = number.as_().into();
+			let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset);
+			apply_state_commit(&mut transaction, commit);
+
+			//finalize an older block
+			if number_u64 > self.finalization_window {
+				let finalizing_hash = if self.finalization_window == 0 {
+					Some(hash)
+				} else {
+					let finalizing = number_u64 - self.finalization_window;
+					if finalizing > self.storage.state_db.best_finalized() {
+						self.blockchain.hash(As::sa(finalizing))?
+					} else {
+						None
+					}
+				};
+				if let Some(finalizing_hash) = finalizing_hash {
+					trace!(target: "db", "Finalizing block #{} ({:?})", number_u64 - self.finalization_window, finalizing_hash);
+					let commit = self.storage.state_db.finalize_block(&finalizing_hash);
+					apply_state_commit(&mut transaction, commit);
+				}
+			}
+
+			debug!(target: "db", "DB Commit {:?} ({}), best = {}", hash, number, pending_block.is_best);
+			self.storage.db.write(transaction).map_err(db_err)?;
+			self.blockchain.update_meta(hash, number, pending_block.is_best);
+		}
+		Ok(())
+	}
+
+	fn revert(&self, n: NumberFor<Block>) -> Result<NumberFor<Block>, client::error::Error> {
+		use client::blockchain::HeaderBackend;
+		let mut best = self.blockchain.info()?.best_number;
+		for c in 0 .. n.as_() {
+			if best == As::sa(0) {
+				return Ok(As::sa(c))
+			}
+			let mut transaction = DBTransaction::new();
+			match self.storage.state_db.revert_one() {
+				Some(commit) => {
+					apply_state_commit(&mut transaction, commit);
+					let removed = self.blockchain.hash(best)?.ok_or_else(
+						|| client::error::ErrorKind::UnknownBlock(
+							format!("Error reverting to {}. Block hash not found.", best)))?;
+					best -= As::sa(1);
+					let key = number_to_db_key(best.clone());
+					let hash = self.blockchain.hash(best)?.ok_or_else(
+						|| client::error::ErrorKind::UnknownBlock(
+							format!("Error reverting to {}. Block hash not found.", best)))?;
+					transaction.put(columns::META, meta_keys::BEST_BLOCK, &key);
+					transaction.delete(columns::BLOCK_INDEX, removed.as_ref());
+					self.storage.db.write(transaction).map_err(db_err)?;
+					self.blockchain.update_meta(hash, best, true);
+				}
+				None => return Ok(As::sa(c))
+			}
+		}
+		Ok(n)
+	}
+
+	fn blockchain(&self) -> &BlockchainDb<Block> {
+		&self.blockchain
+	}
+
+	fn state_at(&self, block: BlockId<Block>) -> Result<Self::State, client::error::Error> {
+		use client::blockchain::HeaderBackend as BcHeaderBackend;
+
+		// special case for genesis initialization
+		match block {
+			BlockId::Hash(h) if h == Default::default() =>
+				return Ok(DbState::with_storage_for_genesis(self.storage.clone())),
+			_ => {}
+		}
+
+		match self.blockchain.header(block) {
+			Ok(Some(ref hdr)) if !self.storage.state_db.is_pruned(hdr.number().as_()) => {
+				let root = H256::from_slice(hdr.state_root().as_ref());
+				Ok(DbState::with_storage(self.storage.clone(), root))
+			},
+			Err(e) => Err(e),
+			_ => Err(client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()),
+		}
+	}
+}
+
+impl<Block> client::backend::LocalBackend<Block, Blake2Hasher, RlpCodec> for Backend<Block>
+where Block: BlockT {}
+
+#[cfg(test)]
+mod tests {
+	use hashdb::HashDB;
+	use super::*;
+	use client::backend::Backend as BTrait;
+	use client::backend::BlockImportOperation as Op;
+	use client::blockchain::HeaderBackend as BlockchainHeaderBackend;
+	use runtime_primitives::testing::{Header, Block as RawBlock};
+
+	type Block = RawBlock<u64>;
+
+	#[test]
+	fn block_hash_inserted_correctly() {
+		let db = Backend::<Block>::new_test(1);
+		for i in 0..10 {
+			assert!(db.blockchain().hash(i).unwrap().is_none());
+
+			{
+				let id = if i == 0 {
+					BlockId::Hash(Default::default())
+				} else {
+					BlockId::Number(i - 1)
+				};
+
+				let mut op = db.begin_operation(id).unwrap();
+				let header = Header {
+					number: i,
+					parent_hash: if i == 0 {
+						Default::default()
+					} else {
+						db.blockchain.hash(i - 1).unwrap().unwrap()
+					},
+					state_root: Default::default(),
+					digest: Default::default(),
+					extrinsics_root: Default::default(),
+				};
+
+				op.set_block_data(
+					header,
+					Some(vec![]),
+					None,
+					true,
+				).unwrap();
+				db.commit_operation(op).unwrap();
+			}
+
+			assert!(db.blockchain().hash(i).unwrap().is_some())
+		}
+	}
+
+	#[test]
+	fn set_state_data() {
+		let db = Backend::<Block>::new_test(2);
+		{
+			let mut op = db.begin_operation(BlockId::Hash(Default::default())).unwrap();
+			let mut header = Header {
+				number: 0,
+				parent_hash: Default::default(),
+				state_root: Default::default(),
+				digest: Default::default(),
+				extrinsics_root: Default::default(),
+			};
+
+			let storage = vec![
+				(vec![1, 3, 5], vec![2, 4, 6]),
+				(vec![1, 2, 3], vec![9, 9, 9]),
+			];
+
+			header.state_root = op.old_state.storage_root(storage
+				.iter()
+				.cloned()
+				.map(|(x, y)| (x, Some(y)))
+			).0.into();
+
+			op.reset_storage(storage.iter().cloned()).unwrap();
+			op.set_block_data(
+				header,
+				Some(vec![]),
+				None,
+				true
+			).unwrap();
+
+			db.commit_operation(op).unwrap();
+
+			let state = db.state_at(BlockId::Number(0)).unwrap();
+
+			assert_eq!(state.storage(&[1, 3, 5]).unwrap(), Some(vec![2, 4, 6]));
+			assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9]));
+			assert_eq!(state.storage(&[5, 5, 5]).unwrap(), None);
+
+		}
+
+		{
+			let mut op = db.begin_operation(BlockId::Number(0)).unwrap();
+			let mut header = Header {
+				number: 1,
+				parent_hash: Default::default(),
+				state_root: Default::default(),
+				digest: Default::default(),
+				extrinsics_root: Default::default(),
+			};
+
+			let storage = vec![
+				(vec![1, 3, 5], None),
+				(vec![5, 5, 5], Some(vec![4, 5, 6])),
+			];
+
+			let (root, overlay) = op.old_state.storage_root(storage.iter().cloned());
+			op.update_storage(overlay).unwrap();
+			header.state_root = root.into();
+
+			op.set_block_data(
+				header,
+				Some(vec![]),
+				None,
+				true
+			).unwrap();
+
+			db.commit_operation(op).unwrap();
+
+			let state = db.state_at(BlockId::Number(1)).unwrap();
+
+			assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None);
+			assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9]));
+			assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6]));
+		}
+	}
+
+	#[test]
+	fn delete_only_when_negative_rc() {
+		let key;
+		let backend = Backend::<Block>::new_test(0);
+
+		let hash = {
+			let mut op = backend.begin_operation(BlockId::Hash(Default::default())).unwrap();
+			let mut header = Header {
+				number: 0,
+				parent_hash: Default::default(),
+				state_root: Default::default(),
+				digest: Default::default(),
+				extrinsics_root: Default::default(),
+			};
+
+			let storage: Vec<(_, _)> = vec![];
+
+			header.state_root = op.old_state.storage_root(storage
+				.iter()
+				.cloned()
+				.map(|(x, y)| (x, Some(y)))
+			).0.into();
+			let hash = header.hash();
+
+			op.reset_storage(storage.iter().cloned()).unwrap();
+
+			key = op.updates.insert(b"hello");
+			op.set_block_data(
+				header,
+				Some(vec![]),
+				None,
+				true
+			).unwrap();
+
+			backend.commit_operation(op).unwrap();
+
+			assert_eq!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().unwrap(), &b"hello"[..]);
+			hash
+		};
+
+		let hash = {
+			let mut op = backend.begin_operation(BlockId::Number(0)).unwrap();
+			let mut header = Header {
+				number: 1,
+				parent_hash: hash,
+				state_root: Default::default(),
+				digest: Default::default(),
+				extrinsics_root: Default::default(),
+			};
+
+			let storage: Vec<(_, _)> = vec![];
+
+			header.state_root = op.old_state.storage_root(storage
+				.iter()
+				.cloned()
+				.map(|(x, y)| (x, Some(y)))
+			).0.into();
+			let hash = header.hash();
+
+			op.updates.insert(b"hello");
+			op.updates.remove(&key);
+			op.set_block_data(
+				header,
+				Some(vec![]),
+				None,
+				true
+			).unwrap();
+
+			backend.commit_operation(op).unwrap();
+
+			assert_eq!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().unwrap(), &b"hello"[..]);
+			hash
+		};
+
+		{
+			let mut op = backend.begin_operation(BlockId::Number(1)).unwrap();
+			let mut header = Header {
+				number: 2,
+				parent_hash: hash,
+				state_root: Default::default(),
+				digest: Default::default(),
+				extrinsics_root: Default::default(),
+			};
+
+			let storage: Vec<(_, _)> = vec![];
+
+			header.state_root = op.old_state.storage_root(storage
+				.iter()
+				.cloned()
+				.map(|(x, y)| (x, Some(y)))
+			).0.into();
+
+			op.updates.remove(&key);
+			op.set_block_data(
+				header,
+				Some(vec![]),
+				None,
+				true
+			).unwrap();
+
+			backend.commit_operation(op).unwrap();
+
+			assert!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().is_none());
+		}
+	}
+}

substrate/core/client/db/src/light.rs

@@ -0,0 +1,392 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! RocksDB-based light client blockchain storage.
+
+use std::sync::Arc;
+use parking_lot::RwLock;
+
+use kvdb::{KeyValueDB, DBTransaction};
+
+use client::blockchain::{BlockStatus, Cache as BlockchainCache,
+	HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo};
+use client::cht;
+use client::error::{ErrorKind as ClientErrorKind, Result as ClientResult};
+use client::light::blockchain::Storage as LightBlockchainStorage;
+use codec::{Decode, Encode};
+use primitives::{AuthorityId, H256, Blake2Hasher};
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash, HashFor,
+	Zero, One, As, NumberFor};
+use cache::DbCache;
+use utils::{meta_keys, Meta, db_err, number_to_db_key, db_key_to_number, open_database,
+	read_db, read_id, read_meta};
+use DatabaseSettings;
+
+pub(crate) mod columns {
+	pub const META: Option<u32> = ::utils::COLUMN_META;
+	pub const BLOCK_INDEX: Option<u32> = Some(1);
+	pub const HEADER: Option<u32> = Some(2);
+	pub const AUTHORITIES: Option<u32> = Some(3);
+	pub const CHT: Option<u32> = Some(4);
+}
+
+/// Keep authorities for last 'AUTHORITIES_ENTRIES_TO_KEEP' blocks.
+pub(crate) const AUTHORITIES_ENTRIES_TO_KEEP: u64 = cht::SIZE;
+
+/// Light blockchain storage. Stores most recent headers + CHTs for older headers.
+pub struct LightStorage<Block: BlockT> {
+	db: Arc<KeyValueDB>,
+	meta: RwLock<Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>>,
+	cache: DbCache<Block>,
+}
+
+#[derive(Clone, PartialEq, Debug)]
+struct BestAuthorities<N> {
+	/// first block, when this set became actual
+	valid_from: N,
+	/// None means that we do not know the set starting from `valid_from` block
+	authorities: Option<Vec<AuthorityId>>,
+}
+
+impl<Block> LightStorage<Block>
+	where
+		Block: BlockT,
+{
+	/// Create new storage with given settings.
+	pub fn new(config: DatabaseSettings) -> ClientResult<Self> {
+		let db = open_database(&config, "light")?;
+
+		Self::from_kvdb(db as Arc<_>)
+	}
+
+	#[cfg(test)]
+	pub(crate) fn new_test() -> Self {
+		use utils::NUM_COLUMNS;
+
+		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
+
+		Self::from_kvdb(db as Arc<_>).expect("failed to create test-db")
+	}
+
+	fn from_kvdb(db: Arc<KeyValueDB>) -> ClientResult<Self> {
+		let cache = DbCache::new(db.clone(), columns::BLOCK_INDEX, columns::AUTHORITIES)?;
+		let meta = RwLock::new(read_meta::<Block>(&*db, columns::HEADER)?);
+
+		Ok(LightStorage {
+			db,
+			meta,
+			cache,
+		})
+	}
+
+	#[cfg(test)]
+	pub(crate) fn db(&self) -> &Arc<KeyValueDB> {
+		&self.db
+	}
+
+	#[cfg(test)]
+	pub(crate) fn cache(&self) -> &DbCache<Block> {
+		&self.cache
+	}
+
+	fn update_meta(&self, hash: Block::Hash, number: <<Block as BlockT>::Header as HeaderT>::Number, is_best: bool) {
+		if is_best {
+			let mut meta = self.meta.write();
+			if number == <<Block as BlockT>::Header as HeaderT>::Number::zero() {
+				meta.genesis_hash = hash;
+			}
+
+			meta.best_number = number;
+			meta.best_hash = hash;
+		}
+	}
+}
+
+impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
+	where
+		Block: BlockT,
+{
+	fn header(&self, id: BlockId<Block>) -> ClientResult<Option<Block::Header>> {
+		match read_db(&*self.db, columns::BLOCK_INDEX, columns::HEADER, id)? {
+			Some(header) => match Block::Header::decode(&mut &header[..]) {
+				Some(header) => Ok(Some(header)),
+				None => return Err(ClientErrorKind::Backend("Error decoding header".into()).into()),
+			}
+			None => Ok(None),
+		}
+	}
+
+	fn info(&self) -> ClientResult<BlockchainInfo<Block>> {
+		let meta = self.meta.read();
+		Ok(BlockchainInfo {
+			best_hash: meta.best_hash,
+			best_number: meta.best_number,
+			genesis_hash: meta.genesis_hash,
+		})
+	}
+
+	fn status(&self, id: BlockId<Block>) -> ClientResult<BlockStatus> {
+		let exists = match id {
+			BlockId::Hash(_) => read_id(&*self.db, columns::BLOCK_INDEX, id)?.is_some(),
+			BlockId::Number(n) => n <= self.meta.read().best_number,
+		};
+		match exists {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	fn number(&self, hash: Block::Hash) -> ClientResult<Option<<<Block as BlockT>::Header as HeaderT>::Number>> {
+		read_id::<Block>(&*self.db, columns::BLOCK_INDEX, BlockId::Hash(hash))
+			.and_then(|key| match key {
+				Some(key) => Ok(Some(db_key_to_number(&key)?)),
+				None => Ok(None),
+			})
+	}
+
+	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Option<Block::Hash>> {
+		read_db::<Block>(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(number)).map(|x|
+			x.map(|raw| HashFor::<Block>::hash(&raw[..])).map(Into::into)
+		)
+	}
+}
+
+impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
+	where
+		Block: BlockT,
+		Block::Hash: From<H256>,
+{
+	fn import_header(&self, is_new_best: bool, header: Block::Header, authorities: Option<Vec<AuthorityId>>) -> ClientResult<()> {
+		let mut transaction = DBTransaction::new();
+
+		let hash = header.hash();
+		let number = *header.number();
+		let key = number_to_db_key(number);
+
+		transaction.put(columns::HEADER, &key, &header.encode());
+		transaction.put(columns::BLOCK_INDEX, hash.as_ref(), &key);
+
+		let best_authorities = if is_new_best {
+			transaction.put(columns::META, meta_keys::BEST_BLOCK, &key);
+
+			// cache authorities for previous block
+			let number: u64 = number.as_();
+			let previous_number = number.checked_sub(1);
+			let best_authorities = previous_number
+				.and_then(|previous_number| self.cache.authorities_at_cache()
+					.commit_best_entry(&mut transaction, As::sa(previous_number), authorities));
+
+			// prune authorities from 'ancient' blocks
+			if let Some(ancient_number) = number.checked_sub(AUTHORITIES_ENTRIES_TO_KEEP) {
+				self.cache.authorities_at_cache().prune_entries(&mut transaction, As::sa(ancient_number))?;
+			}
+
+			best_authorities
+		} else {
+			None
+		};
+
+		// build new CHT if required
+		if let Some(new_cht_number) = cht::is_build_required(cht::SIZE, *header.number()) {
+			let new_cht_start: NumberFor<Block> = cht::start_number(cht::SIZE, new_cht_number);
+			let new_cht_root: Option<Block::Hash> = cht::compute_root::<Block::Header, Blake2Hasher, _>(
+				cht::SIZE, new_cht_number, (new_cht_start.as_()..)
+				.map(|num| self.hash(As::sa(num)).unwrap_or_default()));
+
+			if let Some(new_cht_root) = new_cht_root {
+				transaction.put(columns::CHT, &number_to_db_key(new_cht_start), new_cht_root.as_ref());
+
+				let mut prune_block = new_cht_start;
+				let new_cht_end = cht::end_number(cht::SIZE, new_cht_number);
+				trace!(target: "db", "Replacing blocks [{}..{}] with CHT#{}", new_cht_start, new_cht_end, new_cht_number);
+
+				while prune_block <= new_cht_end {
+					transaction.delete(columns::HEADER, &number_to_db_key(prune_block));
+ 					prune_block += <<Block as BlockT>::Header as HeaderT>::Number::one();
+				}
+			}
+		}
+
+		debug!("Light DB Commit {:?} ({})", hash, number);
+		self.db.write(transaction).map_err(db_err)?;
+		self.update_meta(hash, number, is_new_best);
+		if let Some(best_authorities) = best_authorities {
+			self.cache.authorities_at_cache().update_best_entry(Some(best_authorities));
+		}
+
+		Ok(())
+	}
+
+	fn cht_root(&self, cht_size: u64, block: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Block::Hash> {
+		let no_cht_for_block = || ClientErrorKind::Backend(format!("CHT for block {} not exists", block)).into();
+
+		let cht_number = cht::block_to_cht_number(cht_size, block).ok_or_else(no_cht_for_block)?;
+		let cht_start = cht::start_number(cht_size, cht_number);
+		self.db.get(columns::CHT, &number_to_db_key(cht_start)).map_err(db_err)?
+			.ok_or_else(no_cht_for_block)
+			.and_then(|hash| Block::Hash::decode(&mut &*hash).ok_or_else(no_cht_for_block))
+	}
+
+	fn cache(&self) -> Option<&BlockchainCache<Block>> {
+		Some(&self.cache)
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use client::cht;
+	use runtime_primitives::testing::{H256 as Hash, Header, Block as RawBlock};
+	use super::*;
+
+	type Block = RawBlock<u32>;
+
+	pub fn insert_block(
+		db: &LightStorage<Block>,
+		parent: &Hash,
+		number: u64,
+		authorities: Option<Vec<AuthorityId>>
+	) -> Hash {
+		let header = Header {
+			number: number.into(),
+			parent_hash: *parent,
+			state_root: Default::default(),
+			digest: Default::default(),
+			extrinsics_root: Default::default(),
+		};
+
+		let hash = header.hash();
+		db.import_header(true, header, authorities).unwrap();
+		hash
+	}
+
+	#[test]
+	fn returns_known_header() {
+		let db = LightStorage::new_test();
+		let known_hash = insert_block(&db, &Default::default(), 0, None);
+		let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap();
+		let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap();
+		assert_eq!(header_by_hash, header_by_number);
+	}
+
+	#[test]
+	fn does_not_return_unknown_header() {
+		let db = LightStorage::<Block>::new_test();
+		assert!(db.header(BlockId::Hash(1.into())).unwrap().is_none());
+		assert!(db.header(BlockId::Number(0)).unwrap().is_none());
+	}
+
+	#[test]
+	fn returns_info() {
+		let db = LightStorage::new_test();
+		let genesis_hash = insert_block(&db, &Default::default(), 0, None);
+		let info = db.info().unwrap();
+		assert_eq!(info.best_hash, genesis_hash);
+		assert_eq!(info.best_number, 0);
+		assert_eq!(info.genesis_hash, genesis_hash);
+		let best_hash = insert_block(&db, &genesis_hash, 1, None);
+		let info = db.info().unwrap();
+		assert_eq!(info.best_hash, best_hash);
+		assert_eq!(info.best_number, 1);
+		assert_eq!(info.genesis_hash, genesis_hash);
+	}
+
+	#[test]
+	fn returns_block_status() {
+		let db = LightStorage::new_test();
+		let genesis_hash = insert_block(&db, &Default::default(), 0, None);
+		assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain);
+		assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain);
+		assert_eq!(db.status(BlockId::Hash(1.into())).unwrap(), BlockStatus::Unknown);
+		assert_eq!(db.status(BlockId::Number(1)).unwrap(), BlockStatus::Unknown);
+	}
+
+	#[test]
+	fn returns_block_hash() {
+		let db = LightStorage::new_test();
+		let genesis_hash = insert_block(&db, &Default::default(), 0, None);
+		assert_eq!(db.hash(0).unwrap(), Some(genesis_hash));
+		assert_eq!(db.hash(1).unwrap(), None);
+	}
+
+	#[test]
+	fn import_header_works() {
+		let db = LightStorage::new_test();
+
+		let genesis_hash = insert_block(&db, &Default::default(), 0, None);
+		assert_eq!(db.db.iter(columns::HEADER).count(), 1);
+		assert_eq!(db.db.iter(columns::BLOCK_INDEX).count(), 1);
+
+		let _ = insert_block(&db, &genesis_hash, 1, None);
+		assert_eq!(db.db.iter(columns::HEADER).count(), 2);
+		assert_eq!(db.db.iter(columns::BLOCK_INDEX).count(), 2);
+	}
+
+	#[test]
+	fn ancient_headers_are_replaced_with_cht() {
+		let db = LightStorage::new_test();
+
+		// insert genesis block header (never pruned)
+		let mut prev_hash = insert_block(&db, &Default::default(), 0, None);
+
+		// insert SIZE blocks && ensure that nothing is pruned
+		for number in 0..cht::SIZE {
+			prev_hash = insert_block(&db, &prev_hash, 1 + number, None);
+		}
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 0);
+
+		// insert next SIZE blocks && ensure that nothing is pruned
+		for number in 0..cht::SIZE {
+			prev_hash = insert_block(&db, &prev_hash, 1 + cht::SIZE + number, None);
+		}
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + cht::SIZE) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 0);
+
+		// insert block #{2 * cht::SIZE + 1} && check that new CHT is created + headers of this CHT are pruned
+		insert_block(&db, &prev_hash, 1 + cht::SIZE + cht::SIZE, None);
+		assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + 1) as usize);
+		assert_eq!(db.db.iter(columns::CHT).count(), 1);
+		assert!((0..cht::SIZE).all(|i| db.db.get(columns::HEADER, &number_to_db_key(1 + i)).unwrap().is_none()));
+	}
+
+	#[test]
+	fn get_cht_fails_for_genesis_block() {
+		assert!(LightStorage::<Block>::new_test().cht_root(cht::SIZE, 0).is_err());
+	}
+
+	#[test]
+	fn get_cht_fails_for_non_existant_cht() {
+		assert!(LightStorage::<Block>::new_test().cht_root(cht::SIZE, (cht::SIZE / 2) as u64).is_err());
+	}
+
+	#[test]
+	fn get_cht_works() {
+		let db = LightStorage::new_test();
+
+		// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
+		let mut prev_hash = Default::default();
+		for i in 0..1 + cht::SIZE + cht::SIZE + 1 {
+			prev_hash = insert_block(&db, &prev_hash, i as u64, None);
+		}
+
+		let cht_root_1 = db.cht_root(cht::SIZE, cht::start_number(cht::SIZE, 0)).unwrap();
+		let cht_root_2 = db.cht_root(cht::SIZE, (cht::start_number(cht::SIZE, 0) + cht::SIZE / 2) as u64).unwrap();
+		let cht_root_3 = db.cht_root(cht::SIZE, cht::end_number(cht::SIZE, 0)).unwrap();
+		assert_eq!(cht_root_1, cht_root_2);
+		assert_eq!(cht_root_2, cht_root_3);
+	}
+}

substrate/core/client/db/src/utils.rs

@@ -0,0 +1,174 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Db-based backend utility structures and functions, used by both
+//! full and light storages.
+
+use std::sync::Arc;
+use std::io;
+
+use kvdb::{KeyValueDB, DBTransaction};
+use kvdb_rocksdb::{Database, DatabaseConfig};
+
+use client;
+use codec::Decode;
+use hashdb::DBValue;
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{As, Block as BlockT, Header as HeaderT, Hash, HashFor, Zero};
+use DatabaseSettings;
+
+/// Number of columns in the db. Must be the same for both full && light dbs.
+/// Otherwise RocksDb will fail to open database && check its type.
+pub const NUM_COLUMNS: u32 = 7;
+/// Meta column. The set of keys in the column is shared by full && light storages.
+pub const COLUMN_META: Option<u32> = Some(0);
+
+/// Keys of entries in COLUMN_META.
+pub mod meta_keys {
+	/// Type of storage (full or light).
+	pub const TYPE: &[u8; 4] = b"type";
+	/// Best block key.
+	pub const BEST_BLOCK: &[u8; 4] = b"best";
+	/// Best authorities block key.
+	pub const BEST_AUTHORITIES: &[u8; 4] = b"auth";
+}
+
+/// Database metadata.
+pub struct Meta<N, H> {
+	/// Hash of the best known block.
+	pub best_hash: H,
+	/// Number of the best known block.
+	pub best_number: N,
+	/// Hash of the genesis block.
+	pub genesis_hash: H,
+}
+
+/// Type of block key in the database (LE block number).
+pub type BlockKey = [u8; 4];
+
+/// Convert block number into key (LE representation).
+pub fn number_to_db_key<N>(n: N) -> BlockKey where N: As<u64> {
+	let n: u64 = n.as_();
+	assert!(n & 0xffffffff00000000 == 0);
+
+	[
+		(n >> 24) as u8,
+		((n >> 16) & 0xff) as u8,
+		((n >> 8) & 0xff) as u8,
+		(n & 0xff) as u8
+	]
+}
+
+/// Convert block key into block number.
+pub fn db_key_to_number<N>(key: &[u8]) -> client::error::Result<N> where N: As<u64> {
+	match key.len() {
+		4 => Ok((key[0] as u64) << 24
+			| (key[1] as u64) << 16
+			| (key[2] as u64) << 8
+			| (key[3] as u64)).map(As::sa),
+		_ => Err(client::error::ErrorKind::Backend("Invalid block key".into()).into()),
+	}
+}
+
+/// Maps database error to client error
+pub fn db_err(err: io::Error) -> client::error::Error {
+	use std::error::Error;
+	client::error::ErrorKind::Backend(err.description().into()).into()
+}
+
+/// Open RocksDB database.
+pub fn open_database(config: &DatabaseSettings, db_type: &str) -> client::error::Result<Arc<KeyValueDB>> {
+	let mut db_config = DatabaseConfig::with_columns(Some(NUM_COLUMNS));
+	db_config.memory_budget = config.cache_size;
+	let path = config.path.to_str().ok_or_else(|| client::error::ErrorKind::Backend("Invalid database path".into()))?;
+	let db = Database::open(&db_config, &path).map_err(db_err)?;
+
+	// check database type
+	match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
+		Some(stored_type) => {
+			if db_type.as_bytes() != &*stored_type {
+				return Err(client::error::ErrorKind::Backend(
+					format!("Unexpected database type. Expected: {}", db_type)).into());
+			}
+		},
+		None => {
+			let mut transaction = DBTransaction::new();
+			transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_bytes());
+			db.write(transaction).map_err(db_err)?;
+		},
+	}
+
+	Ok(Arc::new(db))
+}
+
+/// Convert block id to block key, reading number from db if required.
+pub fn read_id<Block>(db: &KeyValueDB, col_index: Option<u32>, id: BlockId<Block>) -> Result<Option<BlockKey>, client::error::Error>
+	where
+		Block: BlockT,
+{
+	match id {
+		BlockId::Hash(h) => db.get(col_index, h.as_ref())
+			.map(|v| v.map(|v| {
+				let mut key: [u8; 4] = [0; 4];
+				key.copy_from_slice(&v);
+				key
+			})).map_err(db_err),
+		BlockId::Number(n) => Ok(Some(number_to_db_key(n))),
+	}
+}
+
+/// Read database column entry for the given block.
+pub fn read_db<Block>(db: &KeyValueDB, col_index: Option<u32>, col: Option<u32>, id: BlockId<Block>) -> client::error::Result<Option<DBValue>>
+	where
+		Block: BlockT,
+{
+	read_id(db, col_index, id).and_then(|key| match key {
+		Some(key) => db.get(col, &key).map_err(db_err),
+		None => Ok(None),
+	})
+}
+
+/// Read meta from the database.
+pub fn read_meta<Block>(db: &KeyValueDB, col_header: Option<u32>) -> Result<Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>, client::error::Error>
+	where
+		Block: BlockT,
+{
+	let genesis_number = <<Block as BlockT>::Header as HeaderT>::Number::zero();
+	let (best_hash, best_number) = if let Some(Some(header)) = db.get(COLUMN_META, meta_keys::BEST_BLOCK).and_then(|id|
+		match id {
+			Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]))),
+			None => Ok(None),
+		}).map_err(db_err)?
+	{
+		let hash = header.hash();
+		debug!("DB Opened blockchain db, best {:?} ({})", hash, header.number());
+		(hash, *header.number())
+	} else {
+		(Default::default(), genesis_number)
+	};
+
+	let genesis_hash = db.get(col_header, &number_to_db_key(genesis_number))
+		.map_err(db_err)?
+		.map(|raw| HashFor::<Block>::hash(&raw[..]))
+		.unwrap_or_default()
+		.into();
+
+	Ok(Meta {
+		best_hash,
+		best_number,
+		genesis_hash,
+	})
+}

substrate/core/client/src/backend.rs

@@ -0,0 +1,107 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot Client data backend
+
+use error;
+use primitives::AuthorityId;
+use runtime_primitives::bft::Justification;
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{Block as BlockT, NumberFor};
+use state_machine::backend::Backend as StateBackend;
+use patricia_trie::NodeCodec;
+use hashdb::Hasher;
+
+/// Block insertion operation. Keeps hold if the inserted block state and data.
+pub trait BlockImportOperation<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>,
+{
+	/// Associated state backend type.
+	type State: StateBackend<H, C>;
+
+	/// Returns pending state. Returns None for backends with locally-unavailable state data.
+	fn state(&self) -> error::Result<Option<&Self::State>>;
+	/// Append block data to the transaction.
+	fn set_block_data(
+		&mut self,
+		header: Block::Header,
+		body: Option<Vec<Block::Extrinsic>>,
+		justification: Option<Justification<Block::Hash>>,
+		is_new_best: bool
+	) -> error::Result<()>;
+
+	/// Append authorities set to the transaction. This is a set of parent block (set which
+	/// has been used to check justification of this block).
+	fn update_authorities(&mut self, authorities: Vec<AuthorityId>);
+	/// Inject storage data into the database.
+	fn update_storage(&mut self, update: <Self::State as StateBackend<H, C>>::Transaction) -> error::Result<()>;
+	/// Inject storage data into the database replacing any existing data.
+	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
+}
+
+/// Client backend. Manages the data layer.
+///
+/// Note on state pruning: while an object from `state_at` is alive, the state
+/// should not be pruned. The backend should internally reference-count
+/// its state objects.
+///
+/// The same applies for live `BlockImportOperation`s: while an import operation building on a parent `P`
+/// is alive, the state for `P` should not be pruned.
+pub trait Backend<Block, H, C>: Send + Sync
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>,
+{
+	/// Associated block insertion operation type.
+	type BlockImportOperation: BlockImportOperation<Block, H, C>;
+	/// Associated blockchain backend type.
+	type Blockchain: ::blockchain::Backend<Block>;
+	/// Associated state backend type.
+	type State: StateBackend<H, C>;
+
+	/// Begin a new block insertion transaction with given parent block id.
+	/// When constructing the genesis, this is called with all-zero hash.
+	fn begin_operation(&self, block: BlockId<Block>) -> error::Result<Self::BlockImportOperation>;
+	/// Commit block insertion.
+	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
+	/// Returns reference to blockchain backend.
+	fn blockchain(&self) -> &Self::Blockchain;
+	/// Returns state backend with post-state of given block.
+	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State>;
+	/// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were
+	/// successfully reverted.
+	fn revert(&self, n: NumberFor<Block>) -> error::Result<NumberFor<Block>>;
+}
+
+/// Mark for all Backend implementations, that are making use of state data, stored locally.
+pub trait LocalBackend<Block, H, C>: Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>,
+{}
+
+/// Mark for all Backend implementations, that are fetching required state data from remote nodes.
+pub trait RemoteBackend<Block, H, C>: Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>,
+{}

substrate/core/client/src/block_builder.rs

@@ -0,0 +1,140 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utility struct to build a block.
+
+use std::vec::Vec;
+use codec::{Decode, Encode};
+use state_machine::{self, native_when_possible};
+use runtime_primitives::traits::{Header as HeaderT, Hash, Block as BlockT, One, HashFor};
+use runtime_primitives::generic::BlockId;
+use {backend, error, Client, CallExecutor};
+use runtime_primitives::{ApplyResult, ApplyOutcome};
+use patricia_trie::NodeCodec;
+use primitives::{Blake2Hasher, RlpCodec};
+use hashdb::Hasher;
+use rlp::Encodable;
+
+/// Utility for building new (valid) blocks from a stream of extrinsics.
+pub struct BlockBuilder<B, E, Block, H, C>
+where
+	B: backend::Backend<Block, H, C>,
+	E: CallExecutor<Block, H, C> + Clone,
+	Block: BlockT,
+	H: Hasher,
+	H::Out: Encodable + Ord,
+	C: NodeCodec<H>,
+{
+	header: <Block as BlockT>::Header,
+	extrinsics: Vec<<Block as BlockT>::Extrinsic>,
+	executor: E,
+	state: B::State,
+	changes: state_machine::OverlayedChanges,
+}
+
+impl<B, E, Block> BlockBuilder<B, E, Block, Blake2Hasher, RlpCodec>
+where
+	B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+	E: CallExecutor<Block, Blake2Hasher, RlpCodec> + Clone,
+	Block: BlockT,
+{
+	/// Create a new instance of builder from the given client, building on the latest block.
+	pub fn new(client: &Client<B, E, Block>) -> error::Result<Self> {
+		client.info().and_then(|i| Self::at_block(&BlockId::Hash(i.chain.best_hash), client))
+	}
+
+	/// Create a new instance of builder from the given client using a particular block's ID to
+	/// build upon.
+	pub fn at_block(block_id: &BlockId<Block>, client: &Client<B, E, Block>) -> error::Result<Self> {
+		let number = client.block_number_from_id(block_id)?
+			.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", block_id)))?
+			+ One::one();
+
+		let parent_hash = client.block_hash_from_id(block_id)?
+			.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", block_id)))?;
+
+		let executor = client.executor().clone();
+		let state = client.state_at(block_id)?;
+		let mut changes = Default::default();
+		let header = <<Block as BlockT>::Header as HeaderT>::new(
+			number,
+			Default::default(),
+			Default::default(),
+			parent_hash,
+			Default::default()
+		);
+
+		executor.call_at_state(&state, &mut changes, "initialise_block", &header.encode(), native_when_possible())?;
+		changes.commit_prospective();
+
+		Ok(BlockBuilder {
+			header,
+			extrinsics: Vec::new(),
+			executor,
+			state,
+			changes,
+		})
+	}
+
+	/// Push onto the block's list of extrinsics. This will ensure the extrinsic
+	/// can be validly executed (by executing it); if it is invalid, it'll be returned along with
+	/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
+	pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> error::Result<()> {
+		match self.executor.call_at_state(&self.state, &mut self.changes, "apply_extrinsic", &xt.encode(), native_when_possible()) {
+			Ok((result, _)) => {
+				match ApplyResult::decode(&mut result.as_slice()) {
+					Some(Ok(ApplyOutcome::Success)) | Some(Ok(ApplyOutcome::Fail)) => {
+						self.extrinsics.push(xt);
+						self.changes.commit_prospective();
+						Ok(())
+					}
+					Some(Err(e)) => {
+						self.changes.discard_prospective();
+						Err(error::ErrorKind::ApplyExtinsicFailed(e).into())
+					}
+					None => {
+						self.changes.discard_prospective();
+						Err(error::ErrorKind::CallResultDecode("apply_extrinsic").into())
+					}
+				}
+			}
+			Err(e) => {
+				self.changes.discard_prospective();
+				Err(e)
+			}
+		}
+	}
+
+	/// Consume the builder to return a valid `Block` containing all pushed extrinsics.
+	pub fn bake(mut self) -> error::Result<Block> {
+		let (output, _) = self.executor.call_at_state(
+			&self.state,
+			&mut self.changes,
+			"finalise_block",
+			&[],
+			native_when_possible(),
+		)?;
+		self.header = <<Block as BlockT>::Header as Decode>::decode(&mut &output[..])
+			.expect("Header came straight out of runtime so must be valid");
+
+		debug_assert_eq!(
+			self.header.extrinsics_root().clone(),
+			HashFor::<Block>::ordered_trie_root(self.extrinsics.iter().map(Encode::encode)),
+		);
+
+		Ok(<Block as BlockT>::new(self.header, self.extrinsics))
+	}
+}

substrate/core/client/src/blockchain.rs

@@ -0,0 +1,92 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot blockchain trait
+
+use primitives::AuthorityId;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor};
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::bft::Justification;
+
+use error::{ErrorKind, Result};
+
+/// Blockchain database header backend. Does not perform any validation.
+pub trait HeaderBackend<Block: BlockT>: Send + Sync {
+	/// Get block header. Returns `None` if block is not found.
+	fn header(&self, id: BlockId<Block>) -> Result<Option<Block::Header>>;
+	/// Get blockchain info.
+	fn info(&self) -> Result<Info<Block>>;
+	/// Get block status.
+	fn status(&self, id: BlockId<Block>) -> Result<BlockStatus>;
+	/// Get block number by hash. Returns `None` if the header is not in the chain.
+	fn number(&self, hash: Block::Hash) -> Result<Option<<<Block as BlockT>::Header as HeaderT>::Number>>;
+	/// Get block hash by number. Returns `None` if the header is not in the chain.
+	fn hash(&self, number: NumberFor<Block>) -> Result<Option<Block::Hash>>;
+
+	/// Get block header. Returns `UnknownBlock` error if block is not found.
+	fn expect_header(&self, id: BlockId<Block>) -> Result<Block::Header> {
+		self.header(id)?.ok_or_else(|| ErrorKind::UnknownBlock(format!("{}", id)).into())
+	}
+}
+
+/// Blockchain database backend. Does not perform any validation.
+pub trait Backend<Block: BlockT>: HeaderBackend<Block> {
+	/// Get block body. Returns `None` if block is not found.
+	fn body(&self, id: BlockId<Block>) -> Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
+	/// Get block justification. Returns `None` if justification does not exist.
+	fn justification(&self, id: BlockId<Block>) -> Result<Option<Justification<Block::Hash>>>;
+
+	/// Returns data cache reference, if it is enabled on this backend.
+	fn cache(&self) -> Option<&Cache<Block>>;
+}
+
+/// Blockchain optional data cache.
+pub trait Cache<Block: BlockT>: Send + Sync {
+	/// Returns the set of authorities, that was active at given block or None if there's no entry in the cache.
+	fn authorities_at(&self, block: BlockId<Block>) -> Option<Vec<AuthorityId>>;
+}
+
+/// Block import outcome
+pub enum ImportResult<E> {
+	/// Imported successfully.
+	Imported,
+	/// Block already exists, skippped.
+	AlreadyInChain,
+	/// Unknown parent.
+	UnknownParent,
+	/// Other errror.
+	Err(E),
+}
+
+/// Blockchain info
+#[derive(Debug)]
+pub struct Info<Block: BlockT> {
+	/// Best block hash.
+	pub best_hash: <<Block as BlockT>::Header as HeaderT>::Hash,
+	/// Best block number.
+	pub best_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	/// Genesis block hash.
+	pub genesis_hash: <<Block as BlockT>::Header as HeaderT>::Hash,
+}
+
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Already in the blockchain.
+	InChain,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}

substrate/core/client/src/call_executor.rs

@@ -0,0 +1,199 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+use std::cmp::Ord;
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::Block as BlockT;
+use state_machine::{self, OverlayedChanges, Ext,
+	CodeExecutor, ExecutionManager, native_when_possible};
+use executor::{RuntimeVersion, RuntimeInfo};
+use patricia_trie::NodeCodec;
+use hashdb::Hasher;
+use rlp::Encodable;
+use codec::Decode;
+use primitives::{Blake2Hasher, RlpCodec};
+
+use backend;
+use error;
+
+/// Information regarding the result of a call.
+#[derive(Debug, Clone)]
+pub struct CallResult {
+	/// The data that was returned from the call.
+	pub return_data: Vec<u8>,
+	/// The changes made to the state by the call.
+	pub changes: OverlayedChanges,
+}
+
+/// Method call executor.
+pub trait CallExecutor<B, H, C>
+where
+	B: BlockT,
+	H: Hasher,
+	H::Out: Ord + Encodable,
+	C: NodeCodec<H>,
+{
+	/// Externalities error type.
+	type Error: state_machine::Error;
+
+	/// Execute a call to a contract on top of state in a block of given hash.
+	///
+	/// No changes are made.
+	fn call(&self,
+		id: &BlockId<B>,
+		method: &str,
+		call_data: &[u8],
+	) -> Result<CallResult, error::Error>;
+
+	/// Extract RuntimeVersion of given block
+	///
+	/// No changes are made.
+	fn runtime_version(&self, id: &BlockId<B>) -> Result<RuntimeVersion, error::Error>;
+
+	/// Execute a call to a contract on top of given state.
+	///
+	/// No changes are made.
+	fn call_at_state<
+		S: state_machine::Backend<H, C>,
+		F: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+	>(&self,
+		state: &S,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8],
+		manager: ExecutionManager<F>
+	) -> Result<(Vec<u8>, S::Transaction), error::Error>;
+
+	/// Execute a call to a contract on top of given state, gathering execution proof.
+	///
+	/// No changes are made.
+	fn prove_at_state<S: state_machine::Backend<H, C>>(&self,
+		state: S,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8]
+	) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error>;
+
+	/// Get runtime version if supported.
+	fn native_runtime_version(&self) -> Option<RuntimeVersion>;
+}
+
+/// Call executor that executes methods locally, querying all required
+/// data from local backend.
+pub struct LocalCallExecutor<B, E> {
+	backend: Arc<B>,
+	executor: E,
+}
+
+impl<B, E> LocalCallExecutor<B, E> {
+	/// Creates new instance of local call executor.
+	pub fn new(backend: Arc<B>, executor: E) -> Self {
+		LocalCallExecutor { backend, executor }
+	}
+}
+
+impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
+	fn clone(&self) -> Self {
+		LocalCallExecutor {
+			backend: self.backend.clone(),
+			executor: self.executor.clone(),
+		}
+	}
+}
+
+impl<B, E, Block> CallExecutor<Block, Blake2Hasher, RlpCodec> for LocalCallExecutor<B, E>
+where
+	B: backend::LocalBackend<Block, Blake2Hasher, RlpCodec>,
+	E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+	Block: BlockT,
+{
+	type Error = E::Error;
+
+	fn call(&self,
+		id: &BlockId<Block>,
+		method: &str,
+		call_data: &[u8],
+	) -> error::Result<CallResult> {
+		let mut changes = OverlayedChanges::default();
+		let (return_data, _) = self.call_at_state(
+			&self.backend.state_at(*id)?,
+			&mut changes,
+			method,
+			call_data,
+			native_when_possible(),
+		)?;
+		Ok(CallResult { return_data, changes })
+	}
+
+	fn runtime_version(&self, id: &BlockId<Block>) -> error::Result<RuntimeVersion> {
+		let mut overlay = OverlayedChanges::default();
+		let state = self.backend.state_at(*id)?;
+		use state_machine::Backend;
+		let code = state.storage(b":code")
+			.map_err(|e| error::ErrorKind::Execution(Box::new(e)))?
+			.ok_or(error::ErrorKind::VersionInvalid)?
+			.to_vec();
+		let heap_pages = state.storage(b":heappages")
+			.map_err(|e| error::ErrorKind::Execution(Box::new(e)))?
+			.and_then(|v| u64::decode(&mut &v[..]))
+			.unwrap_or(8) as usize;
+
+		self.executor.runtime_version(&mut Ext::new(&mut overlay, &state), heap_pages, &code)
+			.ok_or(error::ErrorKind::VersionInvalid.into())
+	}
+
+	fn call_at_state<
+		S: state_machine::Backend<Blake2Hasher, RlpCodec>,
+		F: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>,
+	>(&self,
+		state: &S,
+		changes: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8],
+		manager: ExecutionManager<F>,
+	) -> error::Result<(Vec<u8>, S::Transaction)> {
+		state_machine::execute_using_consensus_failure_handler(
+			state,
+			changes,
+			&self.executor,
+			method,
+			call_data,
+			manager,
+		).map_err(Into::into)
+	}
+
+	fn prove_at_state<S: state_machine::Backend<Blake2Hasher, RlpCodec>>(&self,
+		state: S,
+		changes: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8]
+	) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
+		state_machine::prove_execution(
+			state,
+			changes,
+			&self.executor,
+			method,
+			call_data,
+		)
+		.map(|(result, proof, _)| (result, proof))
+		.map_err(Into::into)
+	}
+
+	fn native_runtime_version(&self) -> Option<RuntimeVersion> {
+		<E as RuntimeInfo>::NATIVE_VERSION
+	}
+}

substrate/core/client/src/cht.rs

@@ -0,0 +1,274 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Canonical hash trie definitions and helper functions.
+//!
+//! Each CHT is a trie mapping block numbers to canonical hash.
+//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in
+//! favor of the trie root. When the "ancient" blocks need to be accessed, we simply
+//! request an inclusion proof of a specific block number against the trie with the
+//! root has. A correct proof implies that the claimed block is identical to the one
+//! we discarded.
+
+use hashdb;
+use heapsize::HeapSizeOf;
+use patricia_trie::NodeCodec;
+use rlp::Encodable;
+use triehash;
+
+use primitives::H256;
+use runtime_primitives::traits::{As, Header as HeaderT, SimpleArithmetic, One};
+use state_machine::backend::InMemory as InMemoryState;
+use state_machine::{prove_read, read_proof_check};
+
+use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult};
+
+/// The size of each CHT. This value is passed to every CHT-related function from
+/// production code. Other values are passed from tests.
+pub const SIZE: u64 = 2048;
+
+/// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized.
+pub fn is_build_required<N>(cht_size: u64, block_num: N) -> Option<N>
+	where
+		N: Clone + SimpleArithmetic,
+{
+	let block_cht_num = block_to_cht_number(cht_size, block_num.clone())?;
+	let two = N::one() + N::one();
+	if block_cht_num < two {
+		return None;
+	}
+	let cht_start = start_number(cht_size, block_cht_num.clone());
+	if cht_start != block_num {
+		return None;
+	}
+
+	Some(block_cht_num - two)
+}
+
+/// Compute a CHT root from an iterator of block hashes. Fails if shorter than
+/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`.
+/// Discards the trie's nodes.
+pub fn compute_root<Header, Hasher, I>(
+	cht_size: u64,
+	cht_num: Header::Number,
+	hashes: I,
+) -> Option<Header::Hash>
+	where
+		Header: HeaderT,
+		Header::Hash: From<Hasher::Out>,
+		Hasher: hashdb::Hasher,
+		Hasher::Out: Ord + Encodable,
+		I: IntoIterator<Item=Option<Header::Hash>>,
+{
+	build_pairs::<Header, I>(cht_size, cht_num, hashes)
+		.map(|pairs| triehash::trie_root::<Hasher, _, _, _>(pairs).into())
+}
+
+/// Build CHT-based header proof.
+pub fn build_proof<Header, Hasher, Codec, I>(
+	cht_size: u64,
+	cht_num: Header::Number,
+	block_num: Header::Number,
+	hashes: I
+) -> Option<Vec<Vec<u8>>>
+	where
+		Header: HeaderT,
+		Hasher: hashdb::Hasher,
+		Hasher::Out: Ord + Encodable + HeapSizeOf,
+		Codec: NodeCodec<Hasher>,
+		I: IntoIterator<Item=Option<Header::Hash>>,
+{
+	let transaction = build_pairs::<Header, I>(cht_size, cht_num, hashes)?
+		.into_iter()
+		.map(|(k, v)| (k, Some(v)))
+		.collect::<Vec<_>>();
+	let storage = InMemoryState::<Hasher, Codec>::default().update(transaction);
+	let (value, proof) = prove_read(storage, &encode_cht_key(block_num)).ok()?;
+	if value.is_none() {
+		None
+	} else {
+		Some(proof)
+	}
+}
+
+/// Check CHT-based header proof.
+pub fn check_proof<Header, Hasher, Codec>(
+	local_root: Header::Hash,
+	local_number: Header::Number,
+	remote_hash: Header::Hash,
+	remote_proof: Vec<Vec<u8>>
+) -> ClientResult<()>
+	where
+		Header: HeaderT,
+		Header::Hash: From<H256>,
+		Hasher: hashdb::Hasher,
+		Hasher::Out: Ord + Encodable + HeapSizeOf + From<Header::Hash>,
+		Codec: NodeCodec<Hasher>,
+{
+	let local_cht_key = encode_cht_key(local_number);
+	let local_cht_value = read_proof_check::<Hasher, Codec>(local_root.into(), remote_proof,
+		&local_cht_key).map_err(|e| ClientError::from(e))?;
+	let local_cht_value = local_cht_value.ok_or_else(|| ClientErrorKind::InvalidHeaderProof)?;
+	let local_hash: Header::Hash = decode_cht_value(&local_cht_value).ok_or_else(|| ClientErrorKind::InvalidHeaderProof)?;
+	match local_hash == remote_hash {
+		true => Ok(()),
+		false => Err(ClientErrorKind::InvalidHeaderProof.into()),
+	}
+}
+
+/// Build pairs for computing CHT.
+fn build_pairs<Header, I>(
+	cht_size: u64,
+	cht_num: Header::Number,
+	hashes: I
+) -> Option<Vec<(Vec<u8>, Vec<u8>)>>
+	where
+		Header: HeaderT,
+		I: IntoIterator<Item=Option<Header::Hash>>,
+{
+	let start_num = start_number(cht_size, cht_num);
+	let mut pairs = Vec::new();
+	let mut hash_number = start_num;
+	for hash in hashes.into_iter().take(cht_size as usize) {
+		pairs.push(hash.map(|hash| (
+			encode_cht_key(hash_number).to_vec(),
+			encode_cht_value(hash)
+		))?);
+		hash_number += Header::Number::one();
+	}
+
+	if pairs.len() as u64 == cht_size {
+		Some(pairs)
+	} else {
+		None
+	}
+}
+
+/// Get the starting block of a given CHT.
+/// CHT 0 includes block 1...SIZE,
+/// CHT 1 includes block SIZE + 1 ... 2*SIZE
+/// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE).
+/// This is because the genesis hash is assumed to be known
+/// and including it would be redundant.
+pub fn start_number<N: SimpleArithmetic>(cht_size: u64, cht_num: N) -> N {
+	(cht_num * As::sa(cht_size)) + N::one()
+}
+
+/// Get the ending block of a given CHT.
+pub fn end_number<N: SimpleArithmetic>(cht_size: u64, cht_num: N) -> N {
+	(cht_num + N::one()) * As::sa(cht_size)
+}
+
+/// Convert a block number to a CHT number.
+/// Returns `None` for `block_num` == 0, `Some` otherwise.
+pub fn block_to_cht_number<N: SimpleArithmetic>(cht_size: u64, block_num: N) -> Option<N> {
+	if block_num == N::zero() {
+		None
+	} else {
+		Some((block_num - N::one()) / As::sa(cht_size))
+	}
+}
+
+/// Convert header number into CHT key.
+pub fn encode_cht_key<N: As<u64>>(number: N) -> Vec<u8> {
+	let number: u64 = number.as_();
+	vec![
+		(number >> 56) as u8,
+		((number >> 48) & 0xff) as u8,
+		((number >> 40) & 0xff) as u8,
+		((number >> 32) & 0xff) as u8,
+		((number >> 24) & 0xff) as u8,
+		((number >> 16) & 0xff) as u8,
+		((number >> 8) & 0xff) as u8,
+		(number & 0xff) as u8
+	]
+}
+
+/// Convert header hash into CHT value.
+fn encode_cht_value<Hash: AsRef<[u8]>>(hash: Hash) -> Vec<u8> {
+	hash.as_ref().to_vec()
+}
+
+/// Convert CHT value into block header hash.
+pub fn decode_cht_value<Hash: From<H256>>(value: &[u8]) -> Option<Hash> {
+	match value.len() {
+		32 => Some(H256::from_slice(&value[0..32]).into()),
+		_ => None,
+	}
+	
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::{Blake2Hasher, RlpCodec};
+	use test_client::runtime::Header;
+	use super::*;
+
+	#[test]
+	fn is_build_required_works() {
+		assert_eq!(is_build_required(SIZE, 0), None);
+		assert_eq!(is_build_required(SIZE, 1), None);
+		assert_eq!(is_build_required(SIZE, SIZE), None);
+		assert_eq!(is_build_required(SIZE, SIZE + 1), None);
+		assert_eq!(is_build_required(SIZE, 2 * SIZE), None);
+		assert_eq!(is_build_required(SIZE, 2 * SIZE + 1), Some(0));
+		assert_eq!(is_build_required(SIZE, 3 * SIZE), None);
+		assert_eq!(is_build_required(SIZE, 3 * SIZE + 1), Some(1));
+	}
+
+	#[test]
+	fn start_number_works() {
+		assert_eq!(start_number(SIZE, 0), 1);
+		assert_eq!(start_number(SIZE, 1), SIZE + 1);
+		assert_eq!(start_number(SIZE, 2), SIZE + SIZE + 1);
+	}
+
+	#[test]
+	fn end_number_works() {
+		assert_eq!(end_number(SIZE, 0), SIZE);
+		assert_eq!(end_number(SIZE, 1), SIZE + SIZE);
+		assert_eq!(end_number(SIZE, 2), SIZE + SIZE + SIZE);
+	}
+
+	#[test]
+	fn build_pairs_fails_when_no_enough_blocks() {
+		assert!(build_pairs::<Header, _>(SIZE, 0, vec![Some(1.into()); SIZE as usize / 2]).is_none());
+	}
+
+	#[test]
+	fn build_pairs_fails_when_missing_block() {
+		assert!(build_pairs::<Header, _>(SIZE, 0, ::std::iter::repeat(Some(1.into())).take(SIZE as usize / 2)
+			.chain(::std::iter::once(None))
+			.chain(::std::iter::repeat(Some(2.into())).take(SIZE as usize / 2 - 1))).is_none());
+	}
+
+	#[test]
+	fn compute_root_works() {
+		assert!(compute_root::<Header, Blake2Hasher, _>(SIZE, 42, vec![Some(1.into()); SIZE as usize]).is_some());
+	}
+
+	#[test]
+	fn build_proof_fails_when_querying_wrong_block() {
+		assert!(build_proof::<Header, Blake2Hasher, RlpCodec, _>(
+			SIZE, 0, (SIZE * 1000) as u64, vec![Some(1.into()); SIZE as usize]).is_none());
+	}
+
+	#[test]
+	fn build_proof_works() {
+		assert!(build_proof::<Header, Blake2Hasher, RlpCodec, _>(
+			SIZE, 0, (SIZE / 2) as u64, vec![Some(1.into()); SIZE as usize]).is_some());
+	}
+}

substrate/core/client/src/client.rs

@@ -0,0 +1,798 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate Client
+
+use std::sync::Arc;
+use futures::sync::mpsc;
+use parking_lot::{Mutex, RwLock};
+use primitives::AuthorityId;
+use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor};
+use runtime_primitives::BuildStorage;
+use substrate_metadata::JsonMetadataDecodable;
+use primitives::{Blake2Hasher, RlpCodec};
+use primitives::storage::{StorageKey, StorageData};
+use codec::{Encode, Decode};
+use state_machine::{
+	Backend as StateBackend, CodeExecutor,
+	ExecutionStrategy, ExecutionManager, prove_read
+};
+
+use backend::{self, BlockImportOperation};
+use blockchain::{self, Info as ChainInfo, Backend as ChainBackend, HeaderBackend as ChainHeaderBackend};
+use call_executor::{CallExecutor, LocalCallExecutor};
+use executor::{RuntimeVersion, RuntimeInfo};
+use notifications::{StorageNotifications, StorageEventStream};
+use {cht, error, in_mem, block_builder, bft, genesis};
+
+/// Type that implements `futures::Stream` of block import events.
+pub type BlockchainEventStream<Block> = mpsc::UnboundedReceiver<BlockImportNotification<Block>>;
+
+/// Substrate Client
+pub struct Client<B, E, Block> where Block: BlockT {
+	backend: Arc<B>,
+	executor: E,
+	storage_notifications: Mutex<StorageNotifications<Block>>,
+	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<BlockImportNotification<Block>>>>,
+	import_lock: Mutex<()>,
+	importing_block: RwLock<Option<Block::Hash>>, // holds the block hash currently being imported. TODO: replace this with block queue
+	execution_strategy: ExecutionStrategy,
+}
+
+/// A source of blockchain evenets.
+pub trait BlockchainEvents<Block: BlockT> {
+	/// Get block import event stream.
+	fn import_notification_stream(&self) -> BlockchainEventStream<Block>;
+
+	/// Get storage changes event stream.
+	///
+	/// Passing `None` as `filter_keys` subscribes to all storage changes.
+	fn storage_changes_notification_stream(&self, filter_keys: Option<&[StorageKey]>) -> error::Result<StorageEventStream<Block::Hash>>;
+}
+
+/// Chain head information.
+pub trait ChainHead<Block: BlockT> {
+	/// Get best block header.
+	fn best_block_header(&self) -> Result<<Block as BlockT>::Header, error::Error>;
+}
+
+/// Fetch block body by ID.
+pub trait BlockBody<Block: BlockT> {
+	/// Get block body by ID. Returns `None` if the body is not stored.
+	fn block_body(&self, id: &BlockId<Block>) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
+}
+
+/// Client info
+// TODO: split queue info from chain info and amalgamate into single struct.
+#[derive(Debug)]
+pub struct ClientInfo<Block: BlockT> {
+	/// Best block hash.
+	pub chain: ChainInfo<Block>,
+	/// Best block number in the queue.
+	pub best_queued_number: Option<<<Block as BlockT>::Header as HeaderT>::Number>,
+	/// Best queued block hash.
+	pub best_queued_hash: Option<Block::Hash>,
+}
+
+/// Block import result.
+#[derive(Debug)]
+pub enum ImportResult {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the import queue.
+	AlreadyQueued,
+	/// Already in the blockchain.
+	AlreadyInChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Block parent is not in the chain.
+	UnknownParent,
+}
+
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the blockchain.
+	InChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}
+
+/// Block data origin.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum BlockOrigin {
+	/// Genesis block built into the client.
+	Genesis,
+	/// Block is part of the initial sync with the network.
+	NetworkInitialSync,
+	/// Block was broadcasted on the network.
+	NetworkBroadcast,
+	/// Block that was received from the network and validated in the consensus process.
+	ConsensusBroadcast,
+	/// Block that was collated by this node.
+	Own,
+	/// Block was imported from a file.
+	File,
+}
+
+/// Summary of an imported block
+#[derive(Clone, Debug)]
+pub struct BlockImportNotification<Block: BlockT> {
+	/// Imported block header hash.
+	pub hash: Block::Hash,
+	/// Imported block origin.
+	pub origin: BlockOrigin,
+	/// Imported block header.
+	pub header: Block::Header,
+	/// Is this the new best block.
+	pub is_new_best: bool,
+}
+
+/// A header paired with a justification which has already been checked.
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct JustifiedHeader<Block: BlockT> {
+	header: <Block as BlockT>::Header,
+	justification: ::bft::Justification<Block::Hash>,
+	authorities: Vec<AuthorityId>,
+}
+
+impl<Block: BlockT> JustifiedHeader<Block> {
+	/// Deconstruct the justified header into parts.
+	pub fn into_inner(self) -> (<Block as BlockT>::Header, ::bft::Justification<Block::Hash>, Vec<AuthorityId>) {
+		(self.header, self.justification, self.authorities)
+	}
+}
+
+/// Create an instance of in-memory client.
+pub fn new_in_mem<E, Block, S>(
+	executor: E,
+	genesis_storage: S,
+) -> error::Result<Client<in_mem::Backend<Block, Blake2Hasher, RlpCodec>, LocalCallExecutor<in_mem::Backend<Block, Blake2Hasher, RlpCodec>, E>, Block>>
+	where
+		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
+		S: BuildStorage,
+		Block: BlockT,
+{
+	let backend = Arc::new(in_mem::Backend::new());
+	let executor = LocalCallExecutor::new(backend.clone(), executor);
+	Client::new(backend, executor, genesis_storage, ExecutionStrategy::NativeWhenPossible)
+}
+
+impl<B, E, Block> Client<B, E, Block> where
+	B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+	E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+	Block: BlockT,
+{
+	/// Creates new Substrate Client with given blockchain and code executor.
+	pub fn new<S: BuildStorage>(
+		backend: Arc<B>,
+		executor: E,
+		build_genesis_storage: S,
+		execution_strategy: ExecutionStrategy,
+	) -> error::Result<Self> {
+		if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() {
+			let genesis_storage = build_genesis_storage.build_storage()?;
+			let genesis_block = genesis::construct_genesis_block::<Block>(&genesis_storage);
+			info!("Initialising Genesis block/state (state: {}, header-hash: {})", genesis_block.header().state_root(), genesis_block.header().hash());
+			let mut op = backend.begin_operation(BlockId::Hash(Default::default()))?;
+			op.reset_storage(genesis_storage.into_iter())?;
+			op.set_block_data(genesis_block.deconstruct().0, Some(vec![]), None, true)?;
+			backend.commit_operation(op)?;
+		}
+		Ok(Client {
+			backend,
+			executor,
+			storage_notifications: Default::default(),
+			import_notification_sinks: Default::default(),
+			import_lock: Default::default(),
+			importing_block: Default::default(),
+			execution_strategy,
+		})
+	}
+
+	/// Get a reference to the state at a given block.
+	pub fn state_at(&self, block: &BlockId<Block>) -> error::Result<B::State> {
+		self.backend.state_at(*block)
+	}
+
+	/// Expose backend reference. To be used in tests only
+	pub fn backend(&self) -> &Arc<B> {
+		&self.backend
+	}
+
+	/// Return single storage entry of contract under given address in state in a block of given hash.
+	pub fn storage(&self, id: &BlockId<Block>, key: &StorageKey) -> error::Result<Option<StorageData>> {
+		Ok(self.state_at(id)?
+			.storage(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
+			.map(StorageData))
+	}
+
+	/// Get the code at a given block.
+	pub fn code_at(&self, id: &BlockId<Block>) -> error::Result<Vec<u8>> {
+		Ok(self.storage(id, &StorageKey(b":code".to_vec()))?
+			.expect("None is returned if there's no value stored for the given key; ':code' key is always defined; qed").0)
+	}
+
+	/// Get the set of authorities at a given block.
+	pub fn authorities_at(&self, id: &BlockId<Block>) -> error::Result<Vec<AuthorityId>> {
+		match self.backend.blockchain().cache().and_then(|cache| cache.authorities_at(*id)) {
+			Some(cached_value) => Ok(cached_value),
+			None => self.executor.call(id, "authorities",&[])
+				.and_then(|r| Vec::<AuthorityId>::decode(&mut &r.return_data[..])
+					.ok_or(error::ErrorKind::AuthLenInvalid.into()))
+		}
+	}
+
+	/// Get the RuntimeVersion at a given block.
+	pub fn runtime_version_at(&self, id: &BlockId<Block>) -> error::Result<RuntimeVersion> {
+		// TODO: Post Poc-2 return an error if version is missing
+		self.executor.runtime_version(id)
+	}
+
+	/// Get call executor reference.
+	pub fn executor(&self) -> &E {
+		&self.executor
+	}
+
+	/// Returns the runtime metadata as JSON.
+	pub fn json_metadata(&self, id: &BlockId<Block>) -> error::Result<String> {
+		self.executor.call(id, "json_metadata",&[])
+			.and_then(|r| Vec::<JsonMetadataDecodable>::decode(&mut &r.return_data[..])
+					  .ok_or("JSON Metadata decoding failed".into()))
+			.and_then(|metadata| {
+				let mut json = metadata.into_iter().enumerate().fold(String::from("{"),
+					|mut json, (i, m)| {
+						if i > 0 {
+							json.push_str(",");
+						}
+						let (mtype, val) = m.into_json_string();
+						json.push_str(&format!(r#" "{}": {}"#, mtype, val));
+						json
+					}
+				);
+				json.push_str(" }");
+
+				Ok(json)
+			})
+	}
+
+	/// Reads storage value at a given block + key, returning read proof.
+	pub fn read_proof(&self, id: &BlockId<Block>, key: &[u8]) -> error::Result<Vec<Vec<u8>>> {
+		self.state_at(id)
+			.and_then(|state| prove_read(state, key)
+				.map(|(_, proof)| proof)
+				.map_err(Into::into))
+	}
+
+	/// Execute a call to a contract on top of state in a block of given hash
+	/// AND returning execution proof.
+	///
+	/// No changes are made.
+	pub fn execution_proof(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> error::Result<(Vec<u8>, Vec<Vec<u8>>)> {
+		self.state_at(id).and_then(|state| self.executor.prove_at_state(state, &mut Default::default(), method, call_data))
+	}
+
+	/// Reads given header and generates CHT-based header proof.
+	pub fn header_proof(&self, id: &BlockId<Block>) -> error::Result<(Block::Header, Vec<Vec<u8>>)> {
+		self.header_proof_with_cht_size(id, cht::SIZE)
+	}
+
+	/// Reads given header and generates CHT-based header proof for CHT of given size.
+	pub fn header_proof_with_cht_size(&self, id: &BlockId<Block>, cht_size: u64) -> error::Result<(Block::Header, Vec<Vec<u8>>)> {
+		let proof_error = || error::ErrorKind::Backend(format!("Failed to generate header proof for {:?}", id));
+		let header = self.header(id)?.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", id)))?;
+		let block_num = *header.number();
+		let cht_num = cht::block_to_cht_number(cht_size, block_num).ok_or_else(proof_error)?;
+		let cht_start = cht::start_number(cht_size, cht_num);
+		let headers = (cht_start.as_()..).map(|num| self.block_hash(As::sa(num)).unwrap_or_default());
+		let proof = cht::build_proof::<Block::Header, Blake2Hasher, RlpCodec, _>(cht_size, cht_num, block_num, headers)
+			.ok_or_else(proof_error)?;
+		Ok((header, proof))
+	}
+
+	/// Create a new block, built on the head of the chain.
+	pub fn new_block(&self) -> error::Result<block_builder::BlockBuilder<B, E, Block, Blake2Hasher, RlpCodec>>
+	where E: Clone
+	{
+		block_builder::BlockBuilder::new(self)
+	}
+
+	/// Create a new block, built on top of `parent`.
+	pub fn new_block_at(&self, parent: &BlockId<Block>) -> error::Result<block_builder::BlockBuilder<B, E, Block, Blake2Hasher, RlpCodec>>
+	where E: Clone
+	{
+		block_builder::BlockBuilder::at_block(parent, &self)
+	}
+
+	/// Set up the native execution environment to call into a native runtime code.
+	pub fn call_api<A, R>(&self, function: &'static str, args: &A) -> error::Result<R>
+		where A: Encode, R: Decode
+	{
+		self.call_api_at(&BlockId::Number(self.info()?.chain.best_number), function, args)
+	}
+
+	/// Call a runtime function at given block.
+	pub fn call_api_at<A, R>(&self, at: &BlockId<Block>, function: &'static str, args: &A) -> error::Result<R>
+		where A: Encode, R: Decode
+	{
+		let parent = at;
+		let header = <<Block as BlockT>::Header as HeaderT>::new(
+			self.block_number_from_id(&parent)?
+				.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", parent)))? + As::sa(1),
+			Default::default(),
+			Default::default(),
+			self.block_hash_from_id(&parent)?
+				.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", parent)))?,
+			Default::default()
+		);
+		self.state_at(&parent).and_then(|state| {
+			let mut overlay = Default::default();
+			let execution_manager = || ExecutionManager::Both(|wasm_result, native_result| {
+				warn!("Consensus error between wasm and native runtime execution at block {:?}", at);
+				warn!("   Function {:?}", function);
+				warn!("   Native result {:?}", native_result);
+				warn!("   Wasm result {:?}", wasm_result);
+				wasm_result
+			});
+			self.executor().call_at_state(
+				&state,
+				&mut overlay,
+				"initialise_block",
+				&header.encode(),
+				execution_manager()
+			)?;
+			let (r, _) = args.using_encoded(|input|
+				self.executor().call_at_state(
+				&state,
+				&mut overlay,
+				function,
+				input,
+				execution_manager()
+			))?;
+			Ok(R::decode(&mut &r[..])
+			   .ok_or_else(|| error::Error::from(error::ErrorKind::CallResultDecode(function)))?)
+		})
+	}
+
+	/// Check a header's justification.
+	pub fn check_justification(
+		&self,
+		header: <Block as BlockT>::Header,
+		justification: ::bft::UncheckedJustification<Block::Hash>,
+	) -> error::Result<JustifiedHeader<Block>> {
+		let parent_hash = header.parent_hash().clone();
+		let authorities = self.authorities_at(&BlockId::Hash(parent_hash))?;
+		let just = ::bft::check_justification::<Block>(&authorities[..], parent_hash, justification)
+			.map_err(|_|
+				error::ErrorKind::BadJustification(
+					format!("{}", header.hash())
+				)
+			)?;
+		Ok(JustifiedHeader {
+			header,
+			justification: just,
+			authorities,
+		})
+	}
+
+	/// Queue a block for import.
+	pub fn import_block(
+		&self,
+		origin: BlockOrigin,
+		header: JustifiedHeader<Block>,
+		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
+	) -> error::Result<ImportResult> {
+		let (header, justification, authorities) = header.into_inner();
+		let parent_hash = header.parent_hash().clone();
+		match self.backend.blockchain().status(BlockId::Hash(parent_hash))? {
+			blockchain::BlockStatus::InChain => {},
+			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+		}
+		let hash = header.hash();
+		let _import_lock = self.import_lock.lock();
+		let height: u64 = header.number().as_();
+		*self.importing_block.write() = Some(hash);
+		let result = self.execute_and_import_block(origin, hash, header, justification, body, authorities);
+		*self.importing_block.write() = None;
+		telemetry!("block.import";
+			"height" => height,
+			"best" => ?hash,
+			"origin" => ?origin
+		);
+		result
+	}
+
+	fn execute_and_import_block(
+		&self,
+		origin: BlockOrigin,
+		hash: Block::Hash,
+		header: Block::Header,
+		justification: bft::Justification<Block::Hash>,
+		body: Option<Vec<Block::Extrinsic>>,
+		authorities: Vec<AuthorityId>,
+	) -> error::Result<ImportResult> {
+		let parent_hash = header.parent_hash().clone();
+		match self.backend.blockchain().status(BlockId::Hash(hash))? {
+			blockchain::BlockStatus::InChain => return Ok(ImportResult::AlreadyInChain),
+			blockchain::BlockStatus::Unknown => {},
+		}
+
+		let mut transaction = self.backend.begin_operation(BlockId::Hash(parent_hash))?;
+		let (storage_update, storage_changes) = match transaction.state()? {
+			Some(transaction_state) => {
+				let mut overlay = Default::default();
+				let mut r = self.executor.call_at_state(
+					transaction_state,
+					&mut overlay,
+					"execute_block",
+					&<Block as BlockT>::new(header.clone(), body.clone().unwrap_or_default()).encode(),
+					match (origin, self.execution_strategy) {
+						(BlockOrigin::NetworkInitialSync, _) | (_, ExecutionStrategy::NativeWhenPossible) =>
+							ExecutionManager::NativeWhenPossible,
+						(_, ExecutionStrategy::AlwaysWasm) => ExecutionManager::AlwaysWasm,
+						_ => ExecutionManager::Both(|wasm_result, native_result| {
+							warn!("Consensus error between wasm and native block execution at block {}", hash);
+							warn!("   Header {:?}", header);
+							warn!("   Native result {:?}", native_result);
+							warn!("   Wasm result {:?}", wasm_result);
+							telemetry!("block.execute.consensus_failure";
+								"hash" => ?hash,
+								"origin" => ?origin,
+								"header" => ?header
+							);
+							wasm_result
+						}),
+					},
+				);
+				let (_, storage_update) = r?;
+				overlay.commit_prospective();
+				(Some(storage_update), Some(overlay.into_committed()))
+			},
+			None => (None, None)
+		};
+
+		let is_new_best = header.number() == &(self.backend.blockchain().info()?.best_number + One::one());
+		trace!("Imported {}, (#{}), best={}, origin={:?}", hash, header.number(), is_new_best, origin);
+		let unchecked: bft::UncheckedJustification<_> = justification.uncheck().into();
+		transaction.set_block_data(header.clone(), body, Some(unchecked.into()), is_new_best)?;
+		transaction.update_authorities(authorities);
+		if let Some(storage_update) = storage_update {
+			transaction.update_storage(storage_update)?;
+		}
+		self.backend.commit_operation(transaction)?;
+
+		if origin == BlockOrigin::NetworkBroadcast || origin == BlockOrigin::Own || origin == BlockOrigin::ConsensusBroadcast {
+
+			if let Some(storage_changes) = storage_changes {
+				// TODO [ToDr] How to handle re-orgs? Should we re-emit all storage changes?
+				self.storage_notifications.lock()
+					.trigger(&hash, storage_changes);
+			}
+
+			let notification = BlockImportNotification::<Block> {
+				hash: hash,
+				origin: origin,
+				header: header,
+				is_new_best: is_new_best,
+			};
+			self.import_notification_sinks.lock()
+				.retain(|sink| sink.unbounded_send(notification.clone()).is_ok());
+		}
+		Ok(ImportResult::Queued)
+	}
+
+	/// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were
+	/// successfully reverted.
+	pub fn revert(&self, n: NumberFor<Block>) -> error::Result<NumberFor<Block>> {
+		Ok(self.backend.revert(n)?)
+	}
+
+	/// Get blockchain info.
+	pub fn info(&self) -> error::Result<ClientInfo<Block>> {
+		let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?;
+		Ok(ClientInfo {
+			chain: info,
+			best_queued_hash: None,
+			best_queued_number: None,
+		})
+	}
+
+	/// Get block status.
+	pub fn block_status(&self, id: &BlockId<Block>) -> error::Result<BlockStatus> {
+		// TODO: more efficient implementation
+		if let BlockId::Hash(ref h) = id {
+			if self.importing_block.read().as_ref().map_or(false, |importing| h == importing) {
+				return Ok(BlockStatus::Queued);
+			}
+		}
+		match self.backend.blockchain().header(*id).map_err(|e| error::Error::from_blockchain(Box::new(e)))?.is_some() {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	/// Get block hash by number.
+	pub fn block_hash(&self, block_number: <<Block as BlockT>::Header as HeaderT>::Number) -> error::Result<Option<Block::Hash>> {
+		self.backend.blockchain().hash(block_number)
+	}
+
+	/// Convert an arbitrary block ID into a block hash.
+	pub fn block_hash_from_id(&self, id: &BlockId<Block>) -> error::Result<Option<Block::Hash>> {
+		match *id {
+			BlockId::Hash(h) => Ok(Some(h)),
+			BlockId::Number(n) => self.block_hash(n),
+		}
+	}
+
+	/// Convert an arbitrary block ID into a block hash.
+	pub fn block_number_from_id(&self, id: &BlockId<Block>) -> error::Result<Option<<<Block as BlockT>::Header as HeaderT>::Number>> {
+		match *id {
+			BlockId::Hash(_) => Ok(self.header(id)?.map(|h| h.number().clone())),
+			BlockId::Number(n) => Ok(Some(n)),
+		}
+	}
+
+	/// Get block header by id.
+	pub fn header(&self, id: &BlockId<Block>) -> error::Result<Option<<Block as BlockT>::Header>> {
+		self.backend.blockchain().header(*id)
+	}
+
+	/// Get block body by id.
+	pub fn body(&self, id: &BlockId<Block>) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
+		self.backend.blockchain().body(*id)
+	}
+
+	/// Get block justification set by id.
+	pub fn justification(&self, id: &BlockId<Block>) -> error::Result<Option<Justification<Block::Hash>>> {
+		self.backend.blockchain().justification(*id)
+	}
+
+	/// Get full block by id.
+	pub fn block(&self, id: &BlockId<Block>) -> error::Result<Option<SignedBlock<Block::Header, Block::Extrinsic, Block::Hash>>> {
+		Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) {
+			(Some(header), Some(extrinsics), Some(justification)) =>
+				Some(SignedBlock { block: RuntimeBlock { header, extrinsics }, justification }),
+			_ => None,
+		})
+	}
+
+	/// Get best block header.
+	pub fn best_block_header(&self) -> error::Result<<Block as BlockT>::Header> {
+		let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?;
+		Ok(self.header(&BlockId::Hash(info.best_hash))?.expect("Best block header must always exist"))
+	}
+}
+
+impl<B, E, Block> bft::BlockImport<Block> for Client<B, E, Block>
+	where
+		B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+		E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+		Block: BlockT,
+{
+	fn import_block(
+		&self,
+		block: Block,
+		justification: ::bft::Justification<Block::Hash>,
+		authorities: &[AuthorityId]
+	) -> bool {
+		let (header, extrinsics) = block.deconstruct();
+		let justified_header = JustifiedHeader {
+			header: header,
+			justification,
+			authorities: authorities.to_vec(),
+		};
+
+		self.import_block(BlockOrigin::ConsensusBroadcast, justified_header, Some(extrinsics)).is_ok()
+	}
+}
+
+impl<B, E, Block> bft::Authorities<Block> for Client<B, E, Block>
+	where
+		B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+		E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+		Block: BlockT,
+{
+	fn authorities(&self, at: &BlockId<Block>) -> Result<Vec<AuthorityId>, bft::Error> {
+		let on_chain_version: Result<_, bft::Error> = self.runtime_version_at(at)
+			.map_err(|e| { trace!("Error getting runtime version {:?}", e); bft::ErrorKind::RuntimeVersionMissing.into() });
+		let on_chain_version = on_chain_version?;
+		let native_version: Result<_, bft::Error> = self.executor.native_runtime_version()
+			.ok_or_else(|| bft::ErrorKind::NativeRuntimeMissing.into());
+		let native_version = native_version?;
+		if !on_chain_version.can_author_with(&native_version) {
+			return Err(bft::ErrorKind::IncompatibleAuthoringRuntime(on_chain_version, native_version).into())
+		}
+		self.authorities_at(at).map_err(|_| {
+			let descriptor = format!("{:?}", at);
+			bft::ErrorKind::StateUnavailable(descriptor).into()
+		})
+	}
+}
+
+impl<B, E, Block> BlockchainEvents<Block> for Client<B, E, Block>
+where
+	E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+	Block: BlockT,
+{
+	/// Get block import event stream.
+	fn import_notification_stream(&self) -> BlockchainEventStream<Block> {
+		let (sink, stream) = mpsc::unbounded();
+		self.import_notification_sinks.lock().push(sink);
+		stream
+	}
+
+	/// Get storage changes event stream.
+	fn storage_changes_notification_stream(&self, filter_keys: Option<&[StorageKey]>) -> error::Result<StorageEventStream<Block::Hash>> {
+		Ok(self.storage_notifications.lock().listen(filter_keys))
+	}
+}
+
+impl<B, E, Block> ChainHead<Block> for Client<B, E, Block>
+where
+	B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+	E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+	Block: BlockT,
+{
+	fn best_block_header(&self) -> error::Result<<Block as BlockT>::Header> {
+		Client::best_block_header(self)
+	}
+}
+
+impl<B, E, Block> BlockBody<Block> for Client<B, E, Block>
+	where
+		B: backend::Backend<Block, Blake2Hasher, RlpCodec>,
+		E: CallExecutor<Block, Blake2Hasher, RlpCodec>,
+		Block: BlockT,
+{
+	fn block_body(&self, id: &BlockId<Block>) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
+		self.body(id)
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use keyring::Keyring;
+	use test_client::{self, TestClient};
+	use test_client::client::BlockOrigin;
+	use test_client::client::backend::Backend as TestBackend;
+	use test_client::BlockBuilderExt;
+	use test_client::runtime::Transfer;
+
+	#[test]
+	fn client_initialises_from_genesis_ok() {
+		let client = test_client::new();
+
+		assert_eq!(client.call_api::<_, u64>("balance_of", &Keyring::Alice.to_raw_public()).unwrap(), 1000);
+		assert_eq!(client.call_api::<_, u64>("balance_of", &Keyring::Ferdie.to_raw_public()).unwrap(), 0);
+	}
+
+	#[test]
+	fn authorities_call_works() {
+		let client = test_client::new();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 0);
+		assert_eq!(client.authorities_at(&BlockId::Number(0)).unwrap(), vec![
+			Keyring::Alice.to_raw_public().into(),
+			Keyring::Bob.to_raw_public().into(),
+			Keyring::Charlie.to_raw_public().into()
+		]);
+	}
+
+	#[test]
+	fn block_builder_works_with_no_transactions() {
+		let client = test_client::new();
+
+		let builder = client.new_block().unwrap();
+
+		client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 1);
+	}
+
+	#[test]
+	fn block_builder_works_with_transactions() {
+		let client = test_client::new();
+
+		let mut builder = client.new_block().unwrap();
+
+		builder.push_transfer(Transfer {
+			from: Keyring::Alice.to_raw_public().into(),
+			to: Keyring::Ferdie.to_raw_public().into(),
+			amount: 42,
+			nonce: 0,
+		}).unwrap();
+
+		client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 1);
+		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
+		assert_eq!(client.call_api::<_, u64>("balance_of", &Keyring::Alice.to_raw_public()).unwrap(), 958);
+		assert_eq!(client.call_api::<_, u64>("balance_of", &Keyring::Ferdie.to_raw_public()).unwrap(), 42);
+	}
+
+	#[test]
+	fn client_uses_authorities_from_blockchain_cache() {
+		let client = test_client::new();
+		test_client::client::in_mem::cache_authorities_at(
+			client.backend().blockchain(),
+			Default::default(),
+			Some(vec![[1u8; 32].into()]));
+		assert_eq!(client.authorities_at(
+			&BlockId::Hash(Default::default())).unwrap(),
+			vec![[1u8; 32].into()]);
+	}
+
+	#[test]
+	fn block_builder_does_not_include_invalid() {
+		let client = test_client::new();
+
+		let mut builder = client.new_block().unwrap();
+
+		builder.push_transfer(Transfer {
+			from: Keyring::Alice.to_raw_public().into(),
+			to: Keyring::Ferdie.to_raw_public().into(),
+			amount: 42,
+			nonce: 0,
+		}).unwrap();
+
+		assert!(builder.push_transfer(Transfer {
+			from: Keyring::Eve.to_raw_public().into(),
+			to: Keyring::Alice.to_raw_public().into(),
+			amount: 42,
+			nonce: 0,
+		}).is_err());
+
+		client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 1);
+		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
+		assert_eq!(client.body(&BlockId::Number(1)).unwrap().unwrap().len(), 1)
+	}
+
+	#[test]
+	fn json_metadata() {
+		let client = test_client::new();
+
+		let mut builder = client.new_block().unwrap();
+
+		builder.push_transfer(Transfer {
+			from: Keyring::Alice.to_raw_public().into(),
+			to: Keyring::Ferdie.to_raw_public().into(),
+			amount: 42,
+			nonce: 0,
+		}).unwrap();
+
+		assert!(builder.push_transfer(Transfer {
+			from: Keyring::Eve.to_raw_public().into(),
+			to: Keyring::Alice.to_raw_public().into(),
+			amount: 42,
+			nonce: 0,
+		}).is_err());
+
+		client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+
+		assert_eq!(
+			client.json_metadata(&BlockId::Number(1)).unwrap(),
+			r#"{ "events": { "name": "Test", "events": { "event": hallo } } }"#
+		);
+	}
+}

substrate/core/client/src/error.rs

@@ -0,0 +1,154 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot client possible errors.
+
+use std;
+use state_machine;
+use runtime_primitives::ApplyError;
+
+error_chain! {
+	errors {
+		/// Backend error.
+		Backend(s: String) {
+			description("Unrecoverable backend error"),
+			display("Backend error: {}", s),
+		}
+
+		/// Unknown block.
+		UnknownBlock(h: String) {
+			description("unknown block"),
+			display("UnknownBlock: {}", &*h),
+		}
+
+		/// Applying extrinsic error.
+		ApplyExtinsicFailed(e: ApplyError) {
+			description("Extrinsic error"),
+			display("Extrinsic error: {:?}", e),
+		}
+
+		/// Execution error.
+		Execution(e: Box<state_machine::Error>) {
+			description("execution error"),
+			display("Execution: {}", e),
+		}
+
+		/// Blockchain error.
+		Blockchain(e: Box<std::error::Error + Send>) {
+			description("Blockchain error"),
+			display("Blockchain: {}", e),
+		}
+
+		/// Invalid state data.
+		AuthLenEmpty {
+			description("authority count state error"),
+			display("Current state of blockchain has no authority count value"),
+		}
+
+		/// Invalid state data.
+		AuthEmpty(i: u32) {
+			description("authority value state error"),
+			display("Current state of blockchain has no authority value for index {}", i),
+		}
+
+		/// Invalid state data.
+		AuthLenInvalid {
+			description("authority count state error"),
+			display("Current state of blockchain has invalid authority count value"),
+		}
+
+		/// Cound not get runtime version.
+		VersionInvalid {
+			description("Runtime version error"),
+			display("On-chain runtime does not specify version"),
+		}
+
+		/// Invalid state data.
+		AuthInvalid(i: u32) {
+			description("authority value state error"),
+			display("Current state of blockchain has invalid authority value for index {}", i),
+		}
+
+		/// Bad justification for header.
+		BadJustification(h: String) {
+			description("bad justification for header"),
+			display("bad justification for header: {}", &*h),
+		}
+
+		/// Not available on light client.
+		NotAvailableOnLightClient {
+			description("not available on light client"),
+			display("This method is not currently available when running in light client mode"),
+		}
+
+		/// Invalid remote header proof.
+		InvalidHeaderProof {
+			description("invalid header proof"),
+			display("Remote node has responded with invalid header proof"),
+		}
+
+		/// Invalid remote execution proof.
+		InvalidExecutionProof {
+			description("invalid execution proof"),
+			display("Remote node has responded with invalid execution proof"),
+		}
+
+		/// Remote fetch has been cancelled.
+		RemoteFetchCancelled {
+			description("remote fetch cancelled"),
+			display("Remote data fetch has been cancelled"),
+		}
+
+		/// Remote fetch has been failed.
+		RemoteFetchFailed {
+			description("remote fetch failed"),
+			display("Remote data fetch has been failed"),
+		}
+
+		/// Error decoding call result.
+		CallResultDecode(method: &'static str) {
+			description("Error decoding call result")
+			display("Error decoding call result of {}", method)
+		}
+	}
+}
+
+// TODO [ToDr] Temporary, state_machine::Error should be a regular error not Box.
+impl From<Box<state_machine::Error>> for Error {
+	fn from(e: Box<state_machine::Error>) -> Self {
+		ErrorKind::Execution(e).into()
+	}
+}
+
+impl From<state_machine::backend::Void> for Error {
+	fn from(e: state_machine::backend::Void) -> Self {
+		match e {}
+	}
+}
+
+impl Error {
+	/// Chain a blockchain error.
+	pub fn from_blockchain(e: Box<std::error::Error + Send>) -> Self {
+		ErrorKind::Blockchain(e).into()
+	}
+
+	/// Chain a state error.
+	pub fn from_state(e: Box<state_machine::Error + Send>) -> Self {
+		ErrorKind::Execution(e).into()
+	}
+}
+
+impl state_machine::Error for Error {}

substrate/core/client/src/genesis.rs

@@ -0,0 +1,202 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tool for creating the genesis block.
+
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, Zero};
+use runtime_primitives::StorageMap;
+
+/// Create a genesis block, given the initial storage.
+pub fn construct_genesis_block<
+	Block: BlockT
+> (
+	storage: &StorageMap
+) -> Block {
+	let state_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(storage.clone().into_iter());
+	let extrinsics_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(::std::iter::empty::<(&[u8], &[u8])>());
+	Block::new(
+		<<Block as BlockT>::Header as HeaderT>::new(
+			Zero::zero(),
+			extrinsics_root,
+			state_root,
+			Default::default(),
+			Default::default()
+		),
+		Default::default()
+	)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use codec::{Encode, Decode, Joiner};
+	use keyring::Keyring;
+	use executor::NativeExecutionDispatch;
+	use state_machine::{execute, OverlayedChanges, ExecutionStrategy};
+	use state_machine::backend::InMemory;
+	use test_client;
+	use test_client::runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
+	use test_client::runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest, Extrinsic};
+	use primitives::{Blake2Hasher, RlpCodec, ed25519::{Public, Pair}};
+
+	native_executor_instance!(Executor, test_client::runtime::api::dispatch, test_client::runtime::VERSION, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
+
+	fn executor() -> ::executor::NativeExecutor<Executor> {
+		NativeExecutionDispatch::new()
+	}
+
+	fn construct_block(backend: &InMemory<Blake2Hasher, RlpCodec>, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec<Transfer>) -> (Vec<u8>, Hash) {
+		use triehash::ordered_trie_root;
+
+		let transactions = txs.into_iter().map(|tx| {
+			let signature = Pair::from(Keyring::from_public(Public::from_raw(tx.from.0)).unwrap())
+				.sign(&tx.encode()).into();
+
+			Extrinsic { transfer: tx, signature }
+		}).collect::<Vec<_>>();
+
+		let extrinsics_root = ordered_trie_root::<Blake2Hasher, _, _>(transactions.iter().map(Encode::encode)).into();
+
+		println!("root before: {:?}", extrinsics_root);
+		let mut header = Header {
+			parent_hash,
+			number,
+			state_root,
+			extrinsics_root,
+			digest: Digest { logs: vec![], },
+		};
+		let hash = header.hash();
+		let mut overlay = OverlayedChanges::default();
+
+		execute(
+			backend,
+			&mut overlay,
+			&executor(),
+			"initialise_block",
+			&header.encode(),
+			ExecutionStrategy::NativeWhenPossible,
+		).unwrap();
+
+		for tx in transactions.iter() {
+			execute(
+				backend,
+				&mut overlay,
+				&executor(),
+				"apply_extrinsic",
+				&tx.encode(),
+				ExecutionStrategy::NativeWhenPossible,
+			).unwrap();
+		}
+
+		let (ret_data, _) = execute(
+			backend,
+			&mut overlay,
+			&executor(),
+			"finalise_block",
+			&[],
+			ExecutionStrategy::NativeWhenPossible,
+		).unwrap();
+		header = Header::decode(&mut &ret_data[..]).unwrap();
+		println!("root after: {:?}", header.extrinsics_root);
+
+		(vec![].and(&Block { header, extrinsics: transactions }), hash)
+	}
+
+	fn block1(genesis_hash: Hash, backend: &InMemory<Blake2Hasher, RlpCodec>) -> (Vec<u8>, Hash) {
+		construct_block(
+			backend,
+			1,
+			genesis_hash,
+			hex!("25e5b37074063ab75c889326246640729b40d0c86932edc527bc80db0e04fe5c").into(),
+			vec![Transfer {
+				from: Keyring::One.to_raw_public().into(),
+				to: Keyring::Two.to_raw_public().into(),
+				amount: 69,
+				nonce: 0,
+			}]
+		)
+	}
+
+	#[test]
+	fn construct_genesis_should_work_with_native() {
+		let mut storage = GenesisConfig::new_simple(
+			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000
+		).genesis_map();
+		let block = construct_genesis_block::<Block>(&storage);
+		let genesis_hash = block.header.hash();
+		storage.extend(additional_storage_with_genesis(&block).into_iter());
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let _ = execute(
+			&backend,
+			&mut overlay,
+			&executor(),
+			"execute_block",
+			&b1data,
+			ExecutionStrategy::NativeWhenPossible,
+		).unwrap();
+	}
+
+	#[test]
+	fn construct_genesis_should_work_with_wasm() {
+		let mut storage = GenesisConfig::new_simple(
+			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 1000
+		).genesis_map();
+		let block = construct_genesis_block::<Block>(&storage);
+		let genesis_hash = block.header.hash();
+		storage.extend(additional_storage_with_genesis(&block).into_iter());
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let _ = execute(
+			&backend,
+			&mut overlay,
+			&executor(),
+			"execute_block",
+			&b1data,
+			ExecutionStrategy::AlwaysWasm,
+		).unwrap();
+	}
+
+	#[test]
+	#[should_panic]
+	fn construct_genesis_with_bad_transaction_should_panic() {
+		let mut storage = GenesisConfig::new_simple(
+			vec![Keyring::One.to_raw_public().into(), Keyring::Two.to_raw_public().into()], 68
+		).genesis_map();
+		let block = construct_genesis_block::<Block>(&storage);
+		let genesis_hash = block.header.hash();
+		storage.extend(additional_storage_with_genesis(&block).into_iter());
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let _ = execute(
+			&backend,
+			&mut overlay,
+			&Executor::new(),
+			"execute_block",
+			&b1data,
+			ExecutionStrategy::NativeWhenPossible,
+		).unwrap();
+	}
+}

substrate/core/client/src/in_mem.rs

@@ -0,0 +1,439 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! In memory client backend
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use parking_lot::RwLock;
+use error;
+use backend;
+use light;
+use primitives::AuthorityId;
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, As};
+use runtime_primitives::bft::Justification;
+use blockchain::{self, BlockStatus};
+use state_machine::backend::{Backend as StateBackend, InMemory};
+use patricia_trie::NodeCodec;
+use hashdb::Hasher;
+use heapsize::HeapSizeOf;
+
+struct PendingBlock<B: BlockT> {
+	block: StoredBlock<B>,
+	is_best: bool,
+}
+
+#[derive(PartialEq, Eq, Clone)]
+enum StoredBlock<B: BlockT> {
+	Header(B::Header, Option<Justification<B::Hash>>),
+	Full(B, Option<Justification<B::Hash>>),
+}
+
+impl<B: BlockT> StoredBlock<B> {
+	fn new(header: B::Header, body: Option<Vec<B::Extrinsic>>, just: Option<Justification<B::Hash>>) -> Self {
+		match body {
+			Some(body) => StoredBlock::Full(B::new(header, body), just),
+			None => StoredBlock::Header(header, just),
+		}
+	}
+
+	fn header(&self) -> &B::Header {
+		match *self {
+			StoredBlock::Header(ref h, _) => h,
+			StoredBlock::Full(ref b, _) => b.header(),
+		}
+	}
+
+	fn justification(&self) -> Option<&Justification<B::Hash>> {
+		match *self {
+			StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref()
+		}
+	}
+
+	fn extrinsics(&self) -> Option<&[B::Extrinsic]> {
+		match *self {
+			StoredBlock::Header(_, _) => None,
+			StoredBlock::Full(ref b, _) => Some(b.extrinsics())
+		}
+	}
+
+	fn into_inner(self) -> (B::Header, Option<Vec<B::Extrinsic>>, Option<Justification<B::Hash>>) {
+		match self {
+			StoredBlock::Header(header, just) => (header, None, just),
+			StoredBlock::Full(block, just) => {
+				let (header, body) = block.deconstruct();
+				(header, Some(body), just)
+			}
+		}
+	}
+}
+
+#[derive(Clone)]
+struct BlockchainStorage<Block: BlockT> {
+	blocks: HashMap<Block::Hash, StoredBlock<Block>>,
+	hashes: HashMap<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
+	best_hash: Block::Hash,
+	best_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	genesis_hash: Block::Hash,
+	cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
+}
+
+/// In-memory blockchain. Supports concurrent reads.
+pub struct Blockchain<Block: BlockT> {
+	storage: Arc<RwLock<BlockchainStorage<Block>>>,
+	cache: Cache<Block>,
+}
+
+struct Cache<Block: BlockT> {
+	storage: Arc<RwLock<BlockchainStorage<Block>>>,
+	authorities_at: RwLock<HashMap<Block::Hash, Option<Vec<AuthorityId>>>>,
+}
+
+impl<Block: BlockT + Clone> Clone for Blockchain<Block> {
+	fn clone(&self) -> Self {
+		let storage = Arc::new(RwLock::new(self.storage.read().clone()));
+		Blockchain {
+			storage: storage.clone(),
+			cache: Cache {
+				storage,
+				authorities_at: RwLock::new(self.cache.authorities_at.read().clone()),
+			},
+		}
+	}
+}
+
+impl<Block: BlockT> Blockchain<Block> {
+	/// Get header hash of given block.
+	pub fn id(&self, id: BlockId<Block>) -> Option<Block::Hash> {
+		match id {
+			BlockId::Hash(h) => Some(h),
+			BlockId::Number(n) => self.storage.read().hashes.get(&n).cloned(),
+		}
+	}
+
+	/// Create new in-memory blockchain storage.
+	pub fn new() -> Blockchain<Block> {
+		let storage = Arc::new(RwLock::new(
+			BlockchainStorage {
+				blocks: HashMap::new(),
+				hashes: HashMap::new(),
+				best_hash: Default::default(),
+				best_number: Zero::zero(),
+				genesis_hash: Default::default(),
+				cht_roots: HashMap::new(),
+			}));
+		Blockchain {
+			storage: storage.clone(),
+			cache: Cache {
+				storage: storage,
+				authorities_at: Default::default(),
+			},
+		}
+	}
+
+	/// Insert a block header and associated data.
+	pub fn insert(
+		&self,
+		hash: Block::Hash,
+		header: <Block as BlockT>::Header,
+		justification: Option<Justification<Block::Hash>>,
+		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
+		is_new_best: bool
+	) {
+		let number = header.number().clone();
+		let mut storage = self.storage.write();
+		storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification));
+		storage.hashes.insert(number, hash.clone());
+		if is_new_best {
+			storage.best_hash = hash.clone();
+			storage.best_number = number.clone();
+		}
+		if number == Zero::zero() {
+			storage.genesis_hash = hash;
+		}
+	}
+
+	/// Compare this blockchain with another in-mem blockchain
+	pub fn equals_to(&self, other: &Self) -> bool {
+		self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks
+	}
+
+	/// Compare canonical chain to other canonical chain.
+	pub fn canon_equals_to(&self, other: &Self) -> bool {
+		let this = self.storage.read();
+		let other = other.storage.read();
+			this.hashes == other.hashes
+			&& this.best_hash == other.best_hash
+			&& this.best_number == other.best_number
+			&& this.genesis_hash == other.genesis_hash
+	}
+
+	/// Insert CHT root.
+	pub fn insert_cht_root(&self, block: NumberFor<Block>, cht_root: Block::Hash) {
+		self.storage.write().cht_roots.insert(block, cht_root);
+	}
+}
+
+impl<Block: BlockT> blockchain::HeaderBackend<Block> for Blockchain<Block> {
+	fn header(&self, id: BlockId<Block>) -> error::Result<Option<<Block as BlockT>::Header>> {
+		Ok(self.id(id).and_then(|hash| {
+			self.storage.read().blocks.get(&hash).map(|b| b.header().clone())
+		}))
+	}
+
+	fn info(&self) -> error::Result<blockchain::Info<Block>> {
+		let storage = self.storage.read();
+		Ok(blockchain::Info {
+			best_hash: storage.best_hash,
+			best_number: storage.best_number,
+			genesis_hash: storage.genesis_hash,
+		})
+	}
+
+	fn status(&self, id: BlockId<Block>) -> error::Result<BlockStatus> {
+		match self.id(id).map_or(false, |hash| self.storage.read().blocks.contains_key(&hash)) {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	fn number(&self, hash: Block::Hash) -> error::Result<Option<NumberFor<Block>>> {
+		Ok(self.storage.read().blocks.get(&hash).map(|b| *b.header().number()))
+	}
+
+	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> error::Result<Option<Block::Hash>> {
+		Ok(self.id(BlockId::Number(number)))
+	}
+}
+
+
+impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
+	fn body(&self, id: BlockId<Block>) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
+		Ok(self.id(id).and_then(|hash| {
+			self.storage.read().blocks.get(&hash)
+				.and_then(|b| b.extrinsics().map(|x| x.to_vec()))
+		}))
+	}
+
+	fn justification(&self, id: BlockId<Block>) -> error::Result<Option<Justification<Block::Hash>>> {
+		Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b|
+			b.justification().map(|x| x.clone()))
+		))
+	}
+
+	fn cache(&self) -> Option<&blockchain::Cache<Block>> {
+		Some(&self.cache)
+	}
+}
+
+impl<Block: BlockT> light::blockchain::Storage<Block> for Blockchain<Block>
+	where
+		Block::Hash: From<[u8; 32]>,
+{
+	fn import_header(
+		&self,
+		is_new_best: bool,
+		header: Block::Header,
+		authorities: Option<Vec<AuthorityId>>
+	) -> error::Result<()> {
+		let hash = header.hash();
+		let parent_hash = *header.parent_hash();
+		self.insert(hash, header, None, None, is_new_best);
+		if is_new_best {
+			self.cache.insert(parent_hash, authorities);
+		}
+		Ok(())
+	}
+
+	fn cht_root(&self, _cht_size: u64, block: NumberFor<Block>) -> error::Result<Block::Hash> {
+		self.storage.read().cht_roots.get(&block).cloned()
+			.ok_or_else(|| error::ErrorKind::Backend(format!("CHT for block {} not exists", block)).into())
+	}
+
+	fn cache(&self) -> Option<&blockchain::Cache<Block>> {
+		Some(&self.cache)
+	}
+}
+
+/// In-memory operation.
+pub struct BlockImportOperation<Block: BlockT, H: Hasher, C: NodeCodec<H>> {
+	pending_block: Option<PendingBlock<Block>>,
+	pending_authorities: Option<Vec<AuthorityId>>,
+	old_state: InMemory<H, C>,
+	new_state: Option<InMemory<H, C>>,
+}
+
+impl<Block, H, C> backend::BlockImportOperation<Block, H, C> for BlockImportOperation<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf,
+{
+	type State = InMemory<H, C>;
+
+	fn state(&self) -> error::Result<Option<&Self::State>> {
+		Ok(Some(&self.old_state))
+	}
+
+	fn set_block_data(
+		&mut self,
+		header: <Block as BlockT>::Header,
+		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
+		justification: Option<Justification<Block::Hash>>,
+		is_new_best: bool
+	) -> error::Result<()> {
+		assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
+		self.pending_block = Some(PendingBlock {
+			block: StoredBlock::new(header, body, justification),
+			is_best: is_new_best,
+		});
+		Ok(())
+	}
+
+	fn update_authorities(&mut self, authorities: Vec<AuthorityId>) {
+		self.pending_authorities = Some(authorities);
+	}
+
+	fn update_storage(&mut self, update: <InMemory<H, C> as StateBackend<H, C>>::Transaction) -> error::Result<()> {
+		self.new_state = Some(self.old_state.update(update));
+		Ok(())
+	}
+
+	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()> {
+		self.new_state = Some(InMemory::from(iter.collect::<HashMap<_, _>>()));
+		Ok(())
+	}
+}
+
+/// In-memory backend. Keeps all states and blocks in memory. Useful for testing.
+pub struct Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>
+{
+	states: RwLock<HashMap<Block::Hash, InMemory<H, C>>>,
+	blockchain: Blockchain<Block>,
+}
+
+impl<Block, H, C> Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	C: NodeCodec<H>
+{
+	/// Create a new instance of in-mem backend.
+	pub fn new() -> Backend<Block, H, C> {
+		Backend {
+			states: RwLock::new(HashMap::new()),
+			blockchain: Blockchain::new(),
+		}
+	}
+}
+
+impl<Block, H, C> backend::Backend<Block, H, C> for Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	H::Out: HeapSizeOf,
+	C: NodeCodec<H> + Send + Sync,
+{
+	type BlockImportOperation = BlockImportOperation<Block, H, C>;
+	type Blockchain = Blockchain<Block>;
+	type State = InMemory<H, C>;
+
+	fn begin_operation(&self, block: BlockId<Block>) -> error::Result<Self::BlockImportOperation> {
+		let state = match block {
+			BlockId::Hash(ref h) if h.clone() == Default::default() => Self::State::default(),
+			_ => self.state_at(block)?,
+		};
+
+		Ok(BlockImportOperation {
+			pending_block: None,
+			pending_authorities: None,
+			old_state: state,
+			new_state: None,
+		})
+	}
+
+	fn commit_operation(&self, operation: Self::BlockImportOperation) -> error::Result<()> {
+		if let Some(pending_block) = operation.pending_block {
+			let old_state = &operation.old_state;
+			let (header, body, justification) = pending_block.block.into_inner();
+			let hash = header.hash();
+			let parent_hash = *header.parent_hash();
+
+			self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
+			self.blockchain.insert(hash, header, justification, body, pending_block.is_best);
+			// dumb implementation - store value for each block
+			if pending_block.is_best {
+				self.blockchain.cache.insert(parent_hash, operation.pending_authorities);
+			}
+		}
+		Ok(())
+	}
+
+	fn blockchain(&self) -> &Self::Blockchain {
+		&self.blockchain
+	}
+
+	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State> {
+		match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) {
+			Some(state) => Ok(state),
+			None => Err(error::ErrorKind::UnknownBlock(format!("{}", block)).into()),
+		}
+	}
+
+	fn revert(&self, _n: NumberFor<Block>) -> error::Result<NumberFor<Block>> {
+		Ok(As::sa(0))
+	}
+}
+
+impl<Block, H, C> backend::LocalBackend<Block, H, C> for Backend<Block, H, C>
+where
+	Block: BlockT,
+	H: Hasher,
+	H::Out: HeapSizeOf,
+	C: NodeCodec<H> + Send + Sync,
+{}
+
+impl<Block: BlockT> Cache<Block> {
+	fn insert(&self, at: Block::Hash, authorities: Option<Vec<AuthorityId>>) {
+		self.authorities_at.write().insert(at, authorities);
+	}
+}
+
+impl<Block: BlockT> blockchain::Cache<Block> for Cache<Block> {
+	fn authorities_at(&self, block: BlockId<Block>) -> Option<Vec<AuthorityId>> {
+		let hash = match block {
+			BlockId::Hash(hash) => hash,
+			BlockId::Number(number) => self.storage.read().hashes.get(&number).cloned()?,
+		};
+
+		self.authorities_at.read().get(&hash).cloned().unwrap_or(None)
+	}
+}
+
+/// Insert authorities entry into in-memory blockchain cache. Extracted as a separate function to use it in tests.
+pub fn cache_authorities_at<Block: BlockT>(
+	blockchain: &Blockchain<Block>,
+	at: Block::Hash,
+	authorities: Option<Vec<AuthorityId>>
+) {
+	blockchain.cache.insert(at, authorities);
+}

substrate/core/client/src/lib.rs

@@ -0,0 +1,69 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate Client and associated logic.
+
+#![warn(missing_docs)]
+#![recursion_limit="128"]
+
+extern crate substrate_bft as bft;
+extern crate parity_codec as codec;
+extern crate substrate_metadata;
+extern crate substrate_primitives as primitives;
+extern crate sr_io as runtime_io;
+extern crate sr_primitives as runtime_primitives;
+extern crate substrate_state_machine as state_machine;
+#[cfg(test)] extern crate substrate_keyring as keyring;
+#[cfg(test)] extern crate substrate_test_client as test_client;
+#[macro_use] extern crate substrate_telemetry;
+#[macro_use] extern crate slog;	// needed until we can reexport `slog_info` from `substrate_telemetry`
+
+extern crate fnv;
+extern crate futures;
+extern crate parking_lot;
+extern crate triehash;
+extern crate patricia_trie;
+extern crate hashdb;
+extern crate rlp;
+extern crate heapsize;
+
+#[macro_use] extern crate error_chain;
+#[macro_use] extern crate log;
+#[cfg_attr(test, macro_use)] extern crate substrate_executor as executor;
+#[cfg(test)] #[macro_use] extern crate hex_literal;
+
+pub mod error;
+pub mod blockchain;
+pub mod backend;
+pub mod cht;
+pub mod in_mem;
+pub mod genesis;
+pub mod block_builder;
+pub mod light;
+mod call_executor;
+mod client;
+mod notifications;
+
+pub use blockchain::Info as ChainInfo;
+pub use call_executor::{CallResult, CallExecutor, LocalCallExecutor};
+pub use client::{
+	new_in_mem,
+	BlockBody, BlockStatus, BlockOrigin, BlockchainEventStream, BlockchainEvents,
+	Client, ClientInfo, ChainHead,
+	ImportResult, JustifiedHeader,
+};
+pub use notifications::{StorageEventStream, StorageChangeSet};
+pub use state_machine::ExecutionStrategy;

substrate/core/client/src/light/backend.rs

@@ -0,0 +1,229 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client backend. Only stores headers and justifications of blocks.
+//! Everything else is requested from full nodes on demand.
+
+use std::sync::{Arc, Weak};
+use futures::{Future, IntoFuture};
+use parking_lot::RwLock;
+
+use primitives::AuthorityId;
+use runtime_primitives::{bft::Justification, generic::BlockId};
+use runtime_primitives::traits::{Block as BlockT, NumberFor};
+use state_machine::{
+	Backend as StateBackend,
+	TrieBackend as StateTrieBackend,
+	TryIntoTrieBackend as TryIntoStateTrieBackend
+};
+
+use backend::{Backend as ClientBackend, BlockImportOperation, RemoteBackend};
+use blockchain::HeaderBackend as BlockchainHeaderBackend;
+use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult};
+use light::blockchain::{Blockchain, Storage as BlockchainStorage};
+use light::fetcher::{Fetcher, RemoteReadRequest};
+use patricia_trie::NodeCodec;
+use hashdb::Hasher;
+
+/// Light client backend.
+pub struct Backend<S, F> {
+	blockchain: Arc<Blockchain<S, F>>,
+}
+
+/// Light block (header and justification) import operation.
+pub struct ImportOperation<Block: BlockT, S, F> {
+	is_new_best: bool,
+	header: Option<Block::Header>,
+	authorities: Option<Vec<AuthorityId>>,
+	_phantom: ::std::marker::PhantomData<(S, F)>,
+}
+
+/// On-demand state.
+pub struct OnDemandState<Block: BlockT, S, F> {
+	fetcher: Weak<F>,
+	blockchain: Weak<Blockchain<S, F>>,
+	block: Block::Hash,
+	cached_header: RwLock<Option<Block::Header>>,
+}
+
+impl<S, F> Backend<S, F> {
+	/// Create new light backend.
+	pub fn new(blockchain: Arc<Blockchain<S, F>>) -> Self {
+		Self { blockchain }
+	}
+
+	/// Get shared blockchain reference.
+	pub fn blockchain(&self) -> &Arc<Blockchain<S, F>> {
+		&self.blockchain
+	}
+}
+
+impl<S, F, Block, H, C> ClientBackend<Block, H, C> for Backend<S, F> where
+	Block: BlockT,
+	S: BlockchainStorage<Block>,
+	F: Fetcher<Block>,
+	H: Hasher,
+	C: NodeCodec<H>,
+{
+	type BlockImportOperation = ImportOperation<Block, S, F>;
+	type Blockchain = Blockchain<S, F>;
+	type State = OnDemandState<Block, S, F>;
+
+	fn begin_operation(&self, _block: BlockId<Block>) -> ClientResult<Self::BlockImportOperation> {
+		Ok(ImportOperation {
+			is_new_best: false,
+			header: None,
+			authorities: None,
+			_phantom: Default::default(),
+		})
+	}
+
+	fn commit_operation(&self, operation: Self::BlockImportOperation) -> ClientResult<()> {
+		let header = operation.header.expect("commit is called after set_block_data; set_block_data sets header; qed");
+		self.blockchain.storage().import_header(operation.is_new_best, header, operation.authorities)
+	}
+
+	fn blockchain(&self) -> &Blockchain<S, F> {
+		&self.blockchain
+	}
+
+	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
+		let block_hash = match block {
+			BlockId::Hash(h) => Some(h),
+			BlockId::Number(n) => self.blockchain.hash(n).unwrap_or_default(),
+		};
+
+		Ok(OnDemandState {
+			fetcher: self.blockchain.fetcher(),
+			blockchain: Arc::downgrade(&self.blockchain),
+			block: block_hash.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", block)))?,
+			cached_header: RwLock::new(None),
+		})
+	}
+
+	fn revert(&self, _n: NumberFor<Block>) -> ClientResult<NumberFor<Block>> {
+		unimplemented!()
+	}
+}
+
+impl<S, F, Block, H, C> RemoteBackend<Block, H, C> for Backend<S, F>
+where
+	Block: BlockT,
+	S: BlockchainStorage<Block>,
+	F: Fetcher<Block>,
+	H: Hasher,
+	C: NodeCodec<H>,
+{}
+
+impl<S, F, Block, H, C> BlockImportOperation<Block, H, C> for ImportOperation<Block, S, F>
+where
+	Block: BlockT,
+	F: Fetcher<Block>,
+	S: BlockchainStorage<Block>,
+	H: Hasher,
+	C: NodeCodec<H>,
+{
+	type State = OnDemandState<Block, S, F>;
+
+	fn state(&self) -> ClientResult<Option<&Self::State>> {
+		// None means 'locally-stateless' backend
+		Ok(None)
+	}
+
+	fn set_block_data(
+		&mut self,
+		header: Block::Header,
+		_body: Option<Vec<Block::Extrinsic>>,
+		_justification: Option<Justification<Block::Hash>>,
+		is_new_best: bool
+	) -> ClientResult<()> {
+		self.is_new_best = is_new_best;
+		self.header = Some(header);
+		Ok(())
+	}
+
+	fn update_authorities(&mut self, authorities: Vec<AuthorityId>) {
+		self.authorities = Some(authorities);
+	}
+
+	fn update_storage(&mut self, _update: <Self::State as StateBackend<H, C>>::Transaction) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
+	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, _iter: I) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+}
+
+impl<Block, S, F, H, C> StateBackend<H, C> for OnDemandState<Block, S, F>
+	where
+		Block: BlockT,
+		S: BlockchainStorage<Block>,
+		F: Fetcher<Block>,
+		H: Hasher,
+		C: NodeCodec<H>,
+{
+	type Error = ClientError;
+	type Transaction = ();
+
+	fn storage(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		let mut header = self.cached_header.read().clone();
+		if header.is_none() {
+			let cached_header = self.blockchain.upgrade()
+				.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", self.block)).into())
+				.and_then(|blockchain| blockchain.expect_header(BlockId::Hash(self.block)))?;
+			header = Some(cached_header.clone());
+			*self.cached_header.write() = Some(cached_header);
+		}
+
+		self.fetcher.upgrade().ok_or(ClientErrorKind::NotAvailableOnLightClient)?
+			.remote_read(RemoteReadRequest {
+				block: self.block,
+				header: header.expect("if block above guarantees that header is_some(); qed"),
+				key: key.to_vec(),
+				retry_count: None,
+			})
+			.into_future().wait()
+	}
+
+	fn for_keys_with_prefix<A: FnMut(&[u8])>(&self, _prefix: &[u8], _action: A) {
+		// whole state is not available on light node
+	}
+
+	fn storage_root<I>(&self, _delta: I) -> (H::Out, Self::Transaction)
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)> {
+		(H::Out::default(), ())
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		// whole state is not available on light node
+		Vec::new()
+	}
+}
+
+impl<Block, S, F, H, C> TryIntoStateTrieBackend<H, C> for OnDemandState<Block, S, F>
+where
+	Block: BlockT,
+	F: Fetcher<Block>,
+	H: Hasher,
+	C: NodeCodec<H>,
+{
+	fn try_into_trie_backend(self) -> Option<StateTrieBackend<H, C>> {
+		None
+	}
+}

substrate/core/client/src/light/blockchain.rs

@@ -0,0 +1,142 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client blockchin backend. Only stores headers and justifications of recent
+//! blocks. CHT roots are stored for headers of ancient blocks.
+
+use std::sync::Weak;
+use futures::{Future, IntoFuture};
+use parking_lot::Mutex;
+
+use primitives::AuthorityId;
+use runtime_primitives::{bft::Justification, generic::BlockId};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
+
+use blockchain::{Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache,
+	HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo};
+use cht;
+use error::{ErrorKind as ClientErrorKind, Result as ClientResult};
+use light::fetcher::{Fetcher, RemoteHeaderRequest};
+
+/// Light client blockchain storage.
+pub trait Storage<Block: BlockT>: BlockchainHeaderBackend<Block> {
+	/// Store new header.
+	fn import_header(
+		&self,
+		is_new_best: bool,
+		header: Block::Header,
+		authorities: Option<Vec<AuthorityId>>
+	) -> ClientResult<()>;
+
+	/// Get CHT root for given block. Fails if the block is not pruned (not a part of any CHT).
+	fn cht_root(&self, cht_size: u64, block: NumberFor<Block>) -> ClientResult<Block::Hash>;
+
+	/// Get storage cache.
+	fn cache(&self) -> Option<&BlockchainCache<Block>>;
+}
+
+/// Light client blockchain.
+pub struct Blockchain<S, F> {
+	fetcher: Mutex<Weak<F>>,
+	storage: S,
+}
+
+impl<S, F> Blockchain<S, F> {
+	/// Create new light blockchain backed with given storage.
+	pub fn new(storage: S) -> Self {
+		Self {
+			fetcher: Mutex::new(Default::default()),
+			storage,
+		}
+	}
+
+	/// Sets fetcher reference.
+	pub fn set_fetcher(&self, fetcher: Weak<F>) {
+		*self.fetcher.lock() = fetcher;
+	}
+
+	/// Get fetcher weak reference.
+	pub fn fetcher(&self) -> Weak<F> {
+		self.fetcher.lock().clone()
+	}
+
+	/// Get storage reference.
+	pub fn storage(&self) -> &S {
+		&self.storage
+	}
+}
+
+impl<S, F, Block> BlockchainHeaderBackend<Block> for Blockchain<S, F> where Block: BlockT, S: Storage<Block>, F: Fetcher<Block> {
+	fn header(&self, id: BlockId<Block>) -> ClientResult<Option<Block::Header>> {
+		match self.storage.header(id)? {
+			Some(header) => Ok(Some(header)),
+			None => {
+				let number = match id {
+					BlockId::Hash(hash) => match self.storage.number(hash)? {
+						Some(number) => number,
+						None => return Ok(None),
+					},
+					BlockId::Number(number) => number,
+				};
+
+				// if the header is from future or genesis (we never prune genesis) => return
+				if number.is_zero() || self.storage.status(BlockId::Number(number))? != BlockStatus::InChain {
+					return Ok(None);
+				}
+
+				self.fetcher().upgrade().ok_or(ClientErrorKind::NotAvailableOnLightClient)?
+					.remote_header(RemoteHeaderRequest {
+						cht_root: self.storage.cht_root(cht::SIZE, number)?,
+						block: number,
+						retry_count: None,
+					})
+					.into_future().wait()
+					.map(Some)
+			}
+		}
+	}
+
+	fn info(&self) -> ClientResult<BlockchainInfo<Block>> {
+		self.storage.info()
+	}
+
+	fn status(&self, id: BlockId<Block>) -> ClientResult<BlockStatus> {
+		self.storage.status(id)
+	}
+
+	fn number(&self, hash: Block::Hash) -> ClientResult<Option<NumberFor<Block>>> {
+		self.storage.number(hash)
+	}
+
+	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Option<Block::Hash>> {
+		self.storage.hash(number)
+	}
+}
+
+impl<S, F, Block> BlockchainBackend<Block> for Blockchain<S, F> where Block: BlockT, S: Storage<Block>, F: Fetcher<Block> {
+	fn body(&self, _id: BlockId<Block>) -> ClientResult<Option<Vec<Block::Extrinsic>>> {
+		// TODO [light]: fetch from remote node
+		Ok(None)
+	}
+
+	fn justification(&self, _id: BlockId<Block>) -> ClientResult<Option<Justification<Block::Hash>>> {
+		Ok(None)
+	}
+
+	fn cache(&self) -> Option<&BlockchainCache<Block>> {
+		self.storage.cache()
+	}
+}

substrate/core/client/src/light/call_executor.rs

@@ -0,0 +1,191 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client call exector. Executes methods on remote full nodes, fetching
+//! execution proof and checking it locally.
+
+use std::sync::Arc;
+use futures::{IntoFuture, Future};
+
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
+use state_machine::{Backend as StateBackend, CodeExecutor, OverlayedChanges,
+	execution_proof_check, ExecutionManager};
+use primitives::H256;
+use patricia_trie::NodeCodec;
+use hashdb::Hasher;
+use rlp::Encodable;
+
+use blockchain::Backend as ChainBackend;
+use call_executor::{CallExecutor, CallResult};
+use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult};
+use light::fetcher::{Fetcher, RemoteCallRequest};
+use executor::RuntimeVersion;
+use codec::Decode;
+use heapsize::HeapSizeOf;
+use std::marker::PhantomData;
+
+/// Call executor that executes methods on remote node, querying execution proof
+/// and checking proof by re-executing locally.
+pub struct RemoteCallExecutor<B, F, H, C> {
+	blockchain: Arc<B>,
+	fetcher: Arc<F>,
+	_hasher: PhantomData<H>,
+	_codec: PhantomData<C>,
+}
+
+impl<B, F, H, C> Clone for RemoteCallExecutor<B, F, H, C> {
+	fn clone(&self) -> Self {
+		RemoteCallExecutor {
+			blockchain: self.blockchain.clone(),
+			fetcher: self.fetcher.clone(),
+			_hasher: Default::default(),
+			_codec: Default::default(),
+		}
+	}
+}
+
+impl<B, F, H, C> RemoteCallExecutor<B, F, H, C> {
+	/// Creates new instance of remote call executor.
+	pub fn new(blockchain: Arc<B>, fetcher: Arc<F>) -> Self {
+		RemoteCallExecutor { blockchain, fetcher, _hasher: PhantomData, _codec: PhantomData }
+	}
+}
+
+impl<B, F, Block, H, C> CallExecutor<Block, H, C> for RemoteCallExecutor<B, F, H, C>
+where
+	Block: BlockT,
+	B: ChainBackend<Block>,
+	F: Fetcher<Block>,
+	H: Hasher,
+	H::Out: Ord + Encodable,
+	C: NodeCodec<H>
+{
+	type Error = ClientError;
+
+	fn call(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> ClientResult<CallResult> {
+		let block_hash = match *id {
+			BlockId::Hash(hash) => hash,
+			BlockId::Number(number) => self.blockchain.hash(number)?
+				.ok_or_else(|| ClientErrorKind::UnknownBlock(format!("{}", number)))?,
+		};
+		let block_header = self.blockchain.expect_header(id.clone())?;
+
+		self.fetcher.remote_call(RemoteCallRequest {
+			block: block_hash,
+			header: block_header,
+			method: method.into(),
+			call_data: call_data.to_vec(),
+			retry_count: None,
+		}).into_future().wait()
+	}
+
+	fn runtime_version(&self, id: &BlockId<Block>) -> ClientResult<RuntimeVersion> {
+		let call_result = self.call(id, "version", &[])?;
+		RuntimeVersion::decode(&mut call_result.return_data.as_slice())
+			.ok_or_else(|| ClientErrorKind::VersionInvalid.into())
+	}
+
+	fn call_at_state<
+		S: StateBackend<H, C>,
+		FF: FnOnce(Result<Vec<u8>, Self::Error>, Result<Vec<u8>, Self::Error>) -> Result<Vec<u8>, Self::Error>
+	>(&self,
+		_state: &S,
+		_changes: &mut OverlayedChanges,
+		_method: &str,
+		_call_data: &[u8],
+		_m: ExecutionManager<FF>
+	) -> ClientResult<(Vec<u8>, S::Transaction)> {
+		Err(ClientErrorKind::NotAvailableOnLightClient.into())
+	}
+
+	fn prove_at_state<S: StateBackend<H, C>>(
+		&self,
+		_state: S,
+		_changes: &mut OverlayedChanges,
+		_method: &str,
+		_call_data: &[u8]
+	) -> ClientResult<(Vec<u8>, Vec<Vec<u8>>)> {
+		Err(ClientErrorKind::NotAvailableOnLightClient.into())
+	}
+
+	fn native_runtime_version(&self) -> Option<RuntimeVersion> {
+		None
+	}
+}
+
+/// Check remote execution proof using given backend.
+pub fn check_execution_proof<Header, E, H, C>(
+	executor: &E,
+	request: &RemoteCallRequest<Header>,
+	remote_proof: Vec<Vec<u8>>
+) -> ClientResult<CallResult>
+	where
+		Header: HeaderT,
+		E: CodeExecutor<H>,
+		H: Hasher,
+		H::Out: Ord + Encodable + HeapSizeOf + From<H256>,
+		C: NodeCodec<H>,
+{
+	let local_state_root = request.header.state_root();
+
+	let mut changes = OverlayedChanges::default();
+	let (local_result, _) = execution_proof_check::<H, C, _>(
+		H256::from_slice(local_state_root.as_ref()).into(),
+		remote_proof,
+		&mut changes,
+		executor,
+		&request.method,
+		&request.call_data)?;
+
+	Ok(CallResult { return_data: local_result, changes })
+}
+
+#[cfg(test)]
+mod tests {
+	use test_client;
+	use executor::NativeExecutionDispatch;
+	use super::*;
+	use primitives::RlpCodec;
+
+	#[test]
+	fn execution_proof_is_generated_and_checked() {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_storage_root = remote_client.state_at(&remote_block_id)
+			.unwrap().storage_root(::std::iter::empty()).0;
+
+		// 'fetch' execution proof from remote node
+		let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1;
+
+		// check remote execution proof locally
+		let local_executor = test_client::LocalExecutor::new();
+		check_execution_proof::<_, _, _, RlpCodec>(&local_executor, &RemoteCallRequest {
+			block: test_client::runtime::Hash::default(),
+			header: test_client::runtime::Header {
+				state_root: remote_block_storage_root.into(),
+				parent_hash: Default::default(),
+				number: 0,
+				extrinsics_root: Default::default(),
+				digest: Default::default(),
+			},
+			method: "authorities".into(),
+			call_data: vec![],
+			retry_count: None,
+		}, remote_execution_proof).unwrap();
+	}
+}

substrate/core/client/src/light/fetcher.rs

@@ -0,0 +1,308 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client data fetcher. Fetches requested data from remote full nodes.
+
+use futures::IntoFuture;
+
+use primitives::H256;
+use hashdb::Hasher;
+use patricia_trie::NodeCodec;
+use rlp::Encodable;
+use heapsize::HeapSizeOf;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
+use state_machine::{CodeExecutor, read_proof_check};
+use std::marker::PhantomData;
+
+use call_executor::CallResult;
+use cht;
+use error::{Error as ClientError, ErrorKind as ClientErrorKind, Result as ClientResult};
+use light::call_executor::check_execution_proof;
+
+/// Remote call request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteCallRequest<Header: HeaderT> {
+	/// Call at state of given block.
+	pub block: Header::Hash,
+	/// Head of block at which call is perormed.
+	pub header: Header,
+	/// Method to call.
+	pub method: String,
+	/// Call data.
+	pub call_data: Vec<u8>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote canonical header request.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+pub struct RemoteHeaderRequest<Header: HeaderT> {
+	/// The root of CHT this block is included in.
+	pub cht_root: Header::Hash,
+	/// Number of the header to query.
+	pub block: Header::Number,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Remote storage read request.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct RemoteReadRequest<Header: HeaderT> {
+	/// Read at state of given block.
+	pub block: Header::Hash,
+	/// Head of block at which read is perormed.
+	pub header: Header,
+	/// Storage key to read.
+	pub key: Vec<u8>,
+	/// Number of times to retry request. None means that default RETRY_COUNT is used.
+	pub retry_count: Option<usize>,
+}
+
+/// Light client data fetcher. Implementations of this trait must check if remote data
+/// is correct (see FetchedDataChecker) and return already checked data.
+pub trait Fetcher<Block: BlockT>: Send + Sync {
+	/// Remote header future.
+	type RemoteHeaderResult: IntoFuture<Item=Block::Header, Error=ClientError>;
+	/// Remote storage read future.
+	type RemoteReadResult: IntoFuture<Item=Option<Vec<u8>>, Error=ClientError>;
+	/// Remote call result future.
+	type RemoteCallResult: IntoFuture<Item=CallResult, Error=ClientError>;
+
+	/// Fetch remote header.
+	fn remote_header(&self, request: RemoteHeaderRequest<Block::Header>) -> Self::RemoteHeaderResult;
+	/// Fetch remote storage value.
+	fn remote_read(&self, request: RemoteReadRequest<Block::Header>) -> Self::RemoteReadResult;
+	/// Fetch remote call result.
+	fn remote_call(&self, request: RemoteCallRequest<Block::Header>) -> Self::RemoteCallResult;
+}
+
+/// Light client remote data checker.
+///
+/// Implementations of this trait should not use any blockchain data except that is
+/// passed to its methods.
+pub trait FetchChecker<Block: BlockT>: Send + Sync {
+	/// Check remote header proof.
+	fn check_header_proof(
+		&self,
+		request: &RemoteHeaderRequest<Block::Header>,
+		header: Option<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Block::Header>;
+	/// Check remote storage read proof.
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Option<Vec<u8>>>;
+	/// Check remote method execution proof.
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<CallResult>;
+}
+
+/// Remote data checker.
+pub struct LightDataChecker<E, H, C> {
+	executor: E,
+	_hasher: PhantomData<H>,
+	_codec: PhantomData<C>,
+}
+
+impl<E, H, C> LightDataChecker<E, H, C> {
+	/// Create new light data checker.
+	pub fn new(executor: E) -> Self {
+		Self {
+			executor, _hasher: PhantomData, _codec: PhantomData
+		}
+	}
+}
+
+impl<E, Block, H, C> FetchChecker<Block> for LightDataChecker<E, H, C>
+	where
+		Block: BlockT,
+		Block::Hash: Into<H::Out> + From<H256>,
+		E: CodeExecutor<H>,
+		H: Hasher,
+		C: NodeCodec<H> + Sync + Send,
+		H::Out: Ord + Encodable + HeapSizeOf + From<Block::Hash> + From<H256>,
+{
+	fn check_header_proof(
+		&self,
+		request: &RemoteHeaderRequest<Block::Header>,
+		remote_header: Option<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Block::Header> {
+		let remote_header = remote_header.ok_or_else(||
+			ClientError::from(ClientErrorKind::InvalidHeaderProof))?;
+		let remote_header_hash = remote_header.hash();
+		cht::check_proof::<Block::Header, H, C>(
+			request.cht_root,
+			request.block,
+			remote_header_hash,
+			remote_proof)
+			.map(|_| remote_header)
+	}
+
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<Option<Vec<u8>>> {
+		let local_state_root = request.header.state_root().clone();
+		read_proof_check::<H, C>(local_state_root.into(), remote_proof, &request.key).map_err(Into::into)
+	}
+
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: Vec<Vec<u8>>
+	) -> ClientResult<CallResult> {
+		check_execution_proof::<_, _, H, C>(&self.executor, request, remote_proof)
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use futures::future::{ok, err, FutureResult};
+	use parking_lot::Mutex;
+	use call_executor::CallResult;
+	use executor::{self, NativeExecutionDispatch};
+	use error::Error as ClientError;
+	use test_client::{self, TestClient, runtime::{Hash, Block, Header}};
+	use test_client::client::BlockOrigin;
+	use in_mem::{Blockchain as InMemoryBlockchain};
+	use light::fetcher::{Fetcher, FetchChecker, LightDataChecker,
+		RemoteCallRequest, RemoteHeaderRequest};
+	use primitives::{Blake2Hasher, RlpCodec};
+	use runtime_primitives::generic::BlockId;
+	use state_machine::Backend;
+	use super::*;
+
+	pub type OkCallFetcher = Mutex<CallResult>;
+
+	impl Fetcher<Block> for OkCallFetcher {
+		type RemoteHeaderResult = FutureResult<Header, ClientError>;
+		type RemoteReadResult = FutureResult<Option<Vec<u8>>, ClientError>;
+		type RemoteCallResult = FutureResult<CallResult, ClientError>;
+
+		fn remote_header(&self, _request: RemoteHeaderRequest<Header>) -> Self::RemoteHeaderResult {
+			err("Not implemented on test node".into())
+		}
+
+		fn remote_read(&self, _request: RemoteReadRequest<Header>) -> Self::RemoteReadResult {
+			err("Not implemented on test node".into())
+		}
+
+		fn remote_call(&self, _request: RemoteCallRequest<Header>) -> Self::RemoteCallResult {
+			ok((*self.lock()).clone())
+		}
+	}
+
+	fn prepare_for_read_proof_check() -> (
+		LightDataChecker<executor::NativeExecutor<test_client::LocalExecutor>, Blake2Hasher, RlpCodec>,
+		Header, Vec<Vec<u8>>, usize)
+	{
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap();
+		let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+		remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap().storage_root(::std::iter::empty()).0.into();
+
+		// 'fetch' read proof from remote node
+		let authorities_len = remote_client.authorities_at(&remote_block_id).unwrap().len();
+		let remote_read_proof = remote_client.read_proof(&remote_block_id, b":auth:len").unwrap();
+
+		// check remote read proof locally
+		let local_storage = InMemoryBlockchain::<Block>::new();
+		local_storage.insert(remote_block_hash, remote_block_header.clone(), None, None, true);
+		let local_executor = test_client::LocalExecutor::new();
+		let local_checker = LightDataChecker::new(local_executor);
+		(local_checker, remote_block_header, remote_read_proof, authorities_len)
+	}
+
+	fn prepare_for_header_proof_check(insert_cht: bool) -> (
+		LightDataChecker<executor::NativeExecutor<test_client::LocalExecutor>, Blake2Hasher, RlpCodec>,
+		Hash, Header, Vec<Vec<u8>>)
+	{
+		// prepare remote client
+		let remote_client = test_client::new();
+		let mut local_headers_hashes = Vec::new();
+		for i in 0..4 {
+			let builder = remote_client.new_block().unwrap();
+			remote_client.justify_and_import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+			local_headers_hashes.push(remote_client.block_hash(i + 1).unwrap());
+		}
+
+		// 'fetch' header proof from remote node
+		let remote_block_id = BlockId::Number(1);
+		let (remote_block_header, remote_header_proof) = remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap();
+
+		// check remote read proof locally
+		let local_storage = InMemoryBlockchain::<Block>::new();
+		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(4, 0, local_headers_hashes.into_iter()).unwrap();
+		if insert_cht {
+			local_storage.insert_cht_root(1, local_cht_root);
+		}
+		let local_executor = test_client::LocalExecutor::new();
+		let local_checker = LightDataChecker::new(local_executor);
+		(local_checker, local_cht_root, remote_block_header, remote_header_proof)
+	}
+
+	#[test]
+	fn storage_read_proof_is_generated_and_checked() {
+		let (local_checker, remote_block_header, remote_read_proof, authorities_len) = prepare_for_read_proof_check();
+		assert_eq!((&local_checker as &FetchChecker<Block>).check_read_proof(&RemoteReadRequest::<Header> {
+			block: remote_block_header.hash(),
+			header: remote_block_header,
+			key: b":auth:len".to_vec(),
+			retry_count: None,
+		}, remote_read_proof).unwrap().unwrap()[0], authorities_len as u8);
+	}
+
+	#[test]
+	fn header_proof_is_generated_and_checked() {
+		let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		assert_eq!((&local_checker as &FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: local_cht_root,
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header);
+	}
+
+	#[test]
+	fn check_header_proof_fails_if_cht_root_is_invalid() {
+		let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		remote_block_header.number = 100;
+		assert!((&local_checker as &FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: Default::default(),
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).is_err());
+	}
+
+	#[test]
+	fn check_header_proof_fails_if_invalid_header_provided() {
+		let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		remote_block_header.number = 100;
+		assert!((&local_checker as &FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: local_cht_root,
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).is_err());
+	}
+}

substrate/core/client/src/light/mod.rs

@@ -0,0 +1,77 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client components.
+
+pub mod backend;
+pub mod blockchain;
+pub mod call_executor;
+pub mod fetcher;
+
+use std::sync::Arc;
+
+use primitives::{Blake2Hasher, RlpCodec};
+use runtime_primitives::BuildStorage;
+use runtime_primitives::traits::Block as BlockT;
+use state_machine::{CodeExecutor, ExecutionStrategy};
+
+use client::Client;
+use error::Result as ClientResult;
+use light::backend::Backend;
+use light::blockchain::{Blockchain, Storage as BlockchainStorage};
+use light::call_executor::RemoteCallExecutor;
+use light::fetcher::{Fetcher, LightDataChecker};
+use hashdb::Hasher;
+use patricia_trie::NodeCodec;
+
+/// Create an instance of light client blockchain backend.
+pub fn new_light_blockchain<B: BlockT, S: BlockchainStorage<B>, F>(storage: S) -> Arc<Blockchain<S, F>> {
+	Arc::new(Blockchain::new(storage))
+}
+
+/// Create an instance of light client backend.
+pub fn new_light_backend<B: BlockT, S: BlockchainStorage<B>, F: Fetcher<B>>(blockchain: Arc<Blockchain<S, F>>, fetcher: Arc<F>) -> Arc<Backend<S, F>> {
+	blockchain.set_fetcher(Arc::downgrade(&fetcher));
+	Arc::new(Backend::new(blockchain))
+}
+
+/// Create an instance of light client.
+pub fn new_light<B, S, F, GS>(
+	backend: Arc<Backend<S, F>>,
+	fetcher: Arc<F>,
+	genesis_storage: GS,
+) -> ClientResult<Client<Backend<S, F>, RemoteCallExecutor<Blockchain<S, F>, F, Blake2Hasher, RlpCodec>, B>>
+	where
+		B: BlockT,
+		S: BlockchainStorage<B>,
+		F: Fetcher<B>,
+		GS: BuildStorage,
+{
+	let executor = RemoteCallExecutor::new(backend.blockchain().clone(), fetcher);
+	Client::new(backend, executor, genesis_storage, ExecutionStrategy::NativeWhenPossible)
+}
+
+/// Create an instance of fetch data checker.
+pub fn new_fetch_checker<E, H, C>(
+	executor: E,
+) -> LightDataChecker<E, H, C>
+	where
+		E: CodeExecutor<H>,
+		H: Hasher,
+		C: NodeCodec<H>,
+{
+	LightDataChecker::new(executor)
+}

substrate/core/client/src/notifications.rs

@@ -0,0 +1,289 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Storage notifications
+
+use std::{
+	collections::{HashSet, HashMap},
+	sync::Arc,
+};
+
+use fnv::{FnvHashSet, FnvHashMap};
+use futures::sync::mpsc;
+use primitives::storage::{StorageKey, StorageData};
+use runtime_primitives::traits::Block as BlockT;
+
+/// Storage change set
+#[derive(Debug)]
+pub struct StorageChangeSet {
+	changes: Arc<Vec<(StorageKey, Option<StorageData>)>>,
+	filter: Option<HashSet<StorageKey>>,
+}
+
+impl StorageChangeSet {
+	/// Convert the change set into iterator over storage items.
+	pub fn iter<'a>(&'a self) -> impl Iterator<Item=&'a (StorageKey, Option<StorageData>)> + 'a {
+		self.changes
+			.iter()
+			.filter(move |&(key, _)| match self.filter {
+				Some(ref filter) => filter.contains(key),
+				None => true,
+			})
+	}
+}
+
+/// Type that implements `futures::Stream` of storage change events.
+pub type StorageEventStream<H> = mpsc::UnboundedReceiver<(H, StorageChangeSet)>;
+
+type SubscriberId = u64;
+
+/// Manages storage listeners.
+#[derive(Debug)]
+pub struct StorageNotifications<Block: BlockT> {
+	next_id: SubscriberId,
+	wildcard_listeners: FnvHashSet<SubscriberId>,
+	listeners: HashMap<StorageKey, FnvHashSet<SubscriberId>>,
+	sinks: FnvHashMap<SubscriberId, (
+		mpsc::UnboundedSender<(Block::Hash, StorageChangeSet)>,
+		Option<HashSet<StorageKey>>,
+	)>,
+}
+
+impl<Block: BlockT> Default for StorageNotifications<Block> {
+	fn default() -> Self {
+		StorageNotifications {
+			next_id: Default::default(),
+			wildcard_listeners: Default::default(),
+			listeners: Default::default(),
+			sinks: Default::default(),
+		}
+	}
+}
+
+impl<Block: BlockT> StorageNotifications<Block> {
+	/// Trigger notification to all listeners.
+	///
+	/// Note the changes are going to be filtered by listener's filter key.
+	/// In fact no event might be sent if clients are not interested in the changes.
+	pub fn trigger(&mut self, hash: &Block::Hash, changeset: impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)>) {
+		let has_wildcard = !self.wildcard_listeners.is_empty();
+
+		// early exit if no listeners
+		if !has_wildcard && self.listeners.is_empty() {
+			return;
+		}
+
+		let mut subscribers = self.wildcard_listeners.clone();
+		let mut changes = Vec::new();
+
+		// Collect subscribers and changes
+		for (k, v) in changeset {
+			let k = StorageKey(k);
+			let listeners = self.listeners.get(&k);
+
+			if let Some(ref listeners) = listeners {
+				subscribers.extend(listeners.iter());
+			}
+
+			if has_wildcard || listeners.is_some() {
+				changes.push((k, v.map(StorageData)));
+			}
+		}
+
+		// Don't send empty notifications
+		if changes.is_empty() {
+			return;
+		}
+
+		let changes = Arc::new(changes);
+		// Trigger the events
+		for subscriber in subscribers {
+			let should_remove = {
+				let &(ref sink, ref filter) = self.sinks.get(&subscriber)
+					.expect("subscribers returned from self.listeners are always in self.sinks; qed");
+				sink.unbounded_send((hash.clone(), StorageChangeSet {
+					changes: changes.clone(),
+					filter: filter.clone(),
+				})).is_err()
+			};
+
+			if should_remove {
+				self.remove_subscriber(subscriber);
+			}
+		}
+	}
+
+	fn remove_subscriber(&mut self, subscriber: SubscriberId) {
+		if let Some((_, filters)) = self.sinks.remove(&subscriber) {
+			match filters {
+				None => {
+					self.wildcard_listeners.remove(&subscriber);
+				},
+				Some(filters) => {
+					for key in filters {
+						let remove_key = match self.listeners.get_mut(&key) {
+							Some(ref mut set) => {
+								set.remove(&subscriber);
+								set.is_empty()
+							},
+							None => false,
+						};
+
+						if remove_key {
+							self.listeners.remove(&key);
+						}
+					}
+				},
+			}
+		}
+	}
+
+	/// Start listening for particular storage keys.
+	pub fn listen(&mut self, filter_keys: Option<&[StorageKey]>) -> StorageEventStream<Block::Hash> {
+		self.next_id += 1;
+
+		// add subscriber for every key
+		let keys = match filter_keys {
+			None => {
+				self.wildcard_listeners.insert(self.next_id);
+				None
+			},
+			Some(keys) => Some(keys.iter().map(|key| {
+				self.listeners
+					.entry(key.clone())
+					.or_insert_with(Default::default)
+					.insert(self.next_id);
+				key.clone()
+			}).collect())
+		};
+
+		// insert sink
+		let (tx, rx) = mpsc::unbounded();
+		self.sinks.insert(self.next_id, (tx, keys));
+		rx
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use runtime_primitives::testing::{H256 as Hash, Block as RawBlock};
+	use super::*;
+	use futures::Stream;
+
+
+	#[cfg(test)]
+	impl From<Vec<(StorageKey, Option<StorageData>)>> for StorageChangeSet {
+		fn from(changes: Vec<(StorageKey, Option<StorageData>)>) -> Self {
+			StorageChangeSet {
+				changes: Arc::new(changes),
+				filter: None,
+			}
+		}
+	}
+
+	#[cfg(test)]
+	impl PartialEq for StorageChangeSet {
+		fn eq(&self, other: &Self) -> bool {
+			self.iter().eq(other.iter())
+		}
+	}
+
+	type Block = RawBlock<Hash>;
+
+	#[test]
+	fn triggering_change_should_notify_wildcard_listeners() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		let mut recv = notifications.listen(None).wait();
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![3], None),
+		];
+		notifications.trigger(&1.into(), changeset.into_iter());
+
+		// then
+		assert_eq!(recv.next().unwrap(), Ok((1.into(), vec![
+			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
+			(StorageKey(vec![3]), None),
+		].into())));
+	}
+
+	#[test]
+	fn should_only_notify_interested_listeners() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		let mut recv1 = notifications.listen(Some(&[StorageKey(vec![1])])).wait();
+		let mut recv2 = notifications.listen(Some(&[StorageKey(vec![2])])).wait();
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![1], None),
+		];
+		notifications.trigger(&1.into(), changeset.into_iter());
+
+		// then
+		assert_eq!(recv1.next().unwrap(), Ok((1.into(), vec![
+			(StorageKey(vec![1]), None),
+		].into())));
+		assert_eq!(recv2.next().unwrap(), Ok((1.into(), vec![
+			(StorageKey(vec![2]), Some(StorageData(vec![3]))),
+		].into())));
+	}
+
+	#[test]
+	fn should_cleanup_subscribers_if_dropped() {
+		// given
+		let mut notifications = StorageNotifications::<Block>::default();
+		{
+			let _recv1 = notifications.listen(Some(&[StorageKey(vec![1])])).wait();
+			let _recv2 = notifications.listen(Some(&[StorageKey(vec![2])])).wait();
+			let _recv3 = notifications.listen(None).wait();
+			assert_eq!(notifications.listeners.len(), 2);
+			assert_eq!(notifications.wildcard_listeners.len(), 1);
+		}
+
+		// when
+		let changeset = vec![
+			(vec![2], Some(vec![3])),
+			(vec![1], None),
+		];
+		notifications.trigger(&1.into(), changeset.into_iter());
+
+		// then
+		assert_eq!(notifications.listeners.len(), 0);
+		assert_eq!(notifications.wildcard_listeners.len(), 0);
+	}
+
+	#[test]
+	fn should_not_send_empty_notifications() {
+		// given
+		let mut recv = {
+			let mut notifications = StorageNotifications::<Block>::default();
+			let recv = notifications.listen(None).wait();
+
+			// when
+			let changeset = vec![];
+			notifications.trigger(&1.into(), changeset.into_iter());
+			recv
+		};
+
+		// then
+		assert_eq!(recv.next(), None);
+	}
+}