Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/src/call_executor.rs

@@ -0,0 +1,279 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell};
+use codec::{Encode, Decode};
+use sr_primitives::{
+	generic::BlockId, traits::Block as BlockT, traits::NumberFor,
+};
+use state_machine::{
+	self, OverlayedChanges, Ext, ExecutionManager, StateMachine, ExecutionStrategy,
+	backend::Backend as _, ChangesTrieTransaction, StorageProof,
+};
+use executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
+use hash_db::Hasher;
+use primitives::{
+	offchain::OffchainExt, H256, Blake2Hasher, NativeOrEncoded, NeverNativeValue,
+	traits::{CodeExecutor, KeystoreExt},
+};
+
+use sr_api::{ProofRecorder, InitializeBlock};
+use client_api::{
+	error, backend, call_executor::CallExecutor,
+};
+
+/// Call executor that executes methods locally, querying all required
+/// data from local backend.
+pub struct LocalCallExecutor<B, E> {
+	backend: Arc<B>,
+	executor: E,
+	keystore: Option<primitives::traits::BareCryptoStorePtr>,
+}
+
+impl<B, E> LocalCallExecutor<B, E> {
+	/// Creates new instance of local call executor.
+	pub fn new(
+		backend: Arc<B>,
+		executor: E,
+		keystore: Option<primitives::traits::BareCryptoStorePtr>,
+	) -> Self {
+		LocalCallExecutor {
+			backend,
+			executor,
+			keystore,
+		}
+	}
+}
+
+impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
+	fn clone(&self) -> Self {
+		LocalCallExecutor {
+			backend: self.backend.clone(),
+			executor: self.executor.clone(),
+			keystore: self.keystore.clone(),
+		}
+	}
+}
+
+impl<B, E, Block> CallExecutor<Block, Blake2Hasher> for LocalCallExecutor<B, E>
+where
+	B: backend::Backend<Block, Blake2Hasher>,
+	E: CodeExecutor + RuntimeInfo,
+	Block: BlockT<Hash=H256>,
+{
+	type Error = E::Error;
+
+	fn call(
+		&self,
+		id: &BlockId<Block>,
+		method: &str,
+		call_data: &[u8],
+		strategy: ExecutionStrategy,
+		side_effects_handler: Option<OffchainExt>,
+	) -> error::Result<Vec<u8>> {
+		let mut changes = OverlayedChanges::default();
+		let state = self.backend.state_at(*id)?;
+		let return_data = StateMachine::new(
+			&state,
+			self.backend.changes_trie_storage(),
+			side_effects_handler,
+			&mut changes,
+			&self.executor,
+			method,
+			call_data,
+			self.keystore.clone().map(KeystoreExt),
+		).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
+			strategy.get_manager(),
+			false,
+			None,
+		)
+		.map(|(result, _, _)| result)?;
+		self.backend.destroy_state(state)?;
+		Ok(return_data.into_encoded())
+	}
+
+	fn contextual_call<
+		'a,
+		IB: Fn() -> error::Result<()>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(
+		&self,
+		initialize_block_fn: IB,
+		at: &BlockId<Block>,
+		method: &str,
+		call_data: &[u8],
+		changes: &RefCell<OverlayedChanges>,
+		initialize_block: InitializeBlock<'a, Block>,
+		execution_manager: ExecutionManager<EM>,
+		native_call: Option<NC>,
+		side_effects_handler: Option<OffchainExt>,
+		recorder: &Option<ProofRecorder<Block>>,
+		enable_keystore: bool,
+	) -> Result<NativeOrEncoded<R>, error::Error> where ExecutionManager<EM>: Clone {
+		match initialize_block {
+			InitializeBlock::Do(ref init_block)
+				if init_block.borrow().as_ref().map(|id| id != at).unwrap_or(true) => {
+				initialize_block_fn()?;
+			},
+			// We don't need to initialize the runtime at a block.
+			_ => {},
+		}
+
+		let keystore = if enable_keystore {
+			self.keystore.clone().map(KeystoreExt)
+		} else {
+			None
+		};
+
+		let mut state = self.backend.state_at(*at)?;
+
+		let result = match recorder {
+			Some(recorder) => {
+				let trie_state = state.as_trie_backend()
+					.ok_or_else(||
+						Box::new(state_machine::ExecutionError::UnableToGenerateProof)
+							as Box<dyn state_machine::Error>
+					)?;
+
+				let backend = state_machine::ProvingBackend::new_with_recorder(
+					trie_state,
+					recorder.clone()
+				);
+
+				StateMachine::new(
+					&backend,
+					self.backend.changes_trie_storage(),
+					side_effects_handler,
+					&mut *changes.borrow_mut(),
+					&self.executor,
+					method,
+					call_data,
+					keystore,
+				)
+				.execute_using_consensus_failure_handler(
+					execution_manager,
+					false,
+					native_call,
+				)
+				.map(|(result, _, _)| result)
+				.map_err(Into::into)
+			}
+			None => StateMachine::new(
+				&state,
+				self.backend.changes_trie_storage(),
+				side_effects_handler,
+				&mut *changes.borrow_mut(),
+				&self.executor,
+				method,
+				call_data,
+				keystore,
+			)
+			.execute_using_consensus_failure_handler(
+				execution_manager,
+				false,
+				native_call,
+			)
+			.map(|(result, _, _)| result)
+		}?;
+		self.backend.destroy_state(state)?;
+		Ok(result)
+	}
+
+	fn runtime_version(&self, id: &BlockId<Block>) -> error::Result<RuntimeVersion> {
+		let mut overlay = OverlayedChanges::default();
+		let state = self.backend.state_at(*id)?;
+
+		let mut ext = Ext::new(
+			&mut overlay,
+			&state,
+			self.backend.changes_trie_storage(),
+			None,
+		);
+		let version = self.executor.runtime_version(&mut ext);
+		self.backend.destroy_state(state)?;
+		version.ok_or(error::Error::VersionInvalid.into())
+	}
+
+	fn call_at_state<
+		S: state_machine::Backend<Blake2Hasher>,
+		F: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>,
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(&self,
+		state: &S,
+		changes: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8],
+		manager: ExecutionManager<F>,
+		native_call: Option<NC>,
+		side_effects_handler: Option<OffchainExt>,
+	) -> error::Result<(
+		NativeOrEncoded<R>,
+		(S::Transaction, <Blake2Hasher as Hasher>::Out),
+		Option<ChangesTrieTransaction<Blake2Hasher, NumberFor<Block>>>,
+	)> {
+		StateMachine::new(
+			state,
+			self.backend.changes_trie_storage(),
+			side_effects_handler,
+			changes,
+			&self.executor,
+			method,
+			call_data,
+			self.keystore.clone().map(KeystoreExt),
+		).execute_using_consensus_failure_handler(
+			manager,
+			true,
+			native_call,
+		)
+		.map(|(result, storage_tx, changes_tx)| (
+			result,
+			storage_tx.expect("storage_tx is always computed when compute_tx is true; qed"),
+			changes_tx,
+		))
+		.map_err(Into::into)
+	}
+
+	fn prove_at_trie_state<S: state_machine::TrieBackendStorage<Blake2Hasher>>(
+		&self,
+		trie_state: &state_machine::TrieBackend<S, Blake2Hasher>,
+		overlay: &mut OverlayedChanges,
+		method: &str,
+		call_data: &[u8]
+	) -> Result<(Vec<u8>, StorageProof), error::Error> {
+		state_machine::prove_execution_on_trie_backend(
+			trie_state,
+			overlay,
+			&self.executor,
+			method,
+			call_data,
+			self.keystore.clone().map(KeystoreExt),
+		)
+		.map_err(Into::into)
+	}
+
+	fn native_runtime_version(&self) -> Option<&NativeVersion> {
+		Some(self.executor.native_version())
+	}
+}

substrate/client/src/cht.rs

@@ -0,0 +1,435 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Canonical hash trie definitions and helper functions.
+//!
+//! Each CHT is a trie mapping block numbers to canonical hash.
+//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in
+//! favor of the trie root. When the "ancient" blocks need to be accessed, we simply
+//! request an inclusion proof of a specific block number against the trie with the
+//! root has. A correct proof implies that the claimed block is identical to the one
+//! we discarded.
+
+use hash_db;
+use codec::Encode;
+use trie;
+
+use primitives::{H256, convert_hash};
+use sr_primitives::traits::{Header as HeaderT, SimpleArithmetic, Zero, One};
+use state_machine::backend::InMemory as InMemoryState;
+use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, StorageProof,
+	prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend};
+
+use client_api::error::{Error as ClientError, Result as ClientResult};
+
+/// The size of each CHT. This value is passed to every CHT-related function from
+/// production code. Other values are passed from tests.
+const SIZE: u32 = 2048;
+
+/// Gets default CHT size.
+pub fn size<N: From<u32>>() -> N {
+	SIZE.into()
+}
+
+/// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized.
+pub fn is_build_required<N>(cht_size: N, block_num: N) -> Option<N>
+	where
+		N: Clone + SimpleArithmetic,
+{
+	let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?;
+	let two = N::one() + N::one();
+	if block_cht_num < two {
+		return None;
+	}
+	let cht_start = start_number(cht_size, block_cht_num.clone());
+	if cht_start != block_num {
+		return None;
+	}
+
+	Some(block_cht_num - two)
+}
+
+/// Compute a CHT root from an iterator of block hashes. Fails if shorter than
+/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`.
+/// Discards the trie's nodes.
+pub fn compute_root<Header, Hasher, I>(
+	cht_size: Header::Number,
+	cht_num: Header::Number,
+	hashes: I,
+) -> ClientResult<Hasher::Out>
+	where
+		Header: HeaderT,
+		Hasher: hash_db::Hasher,
+		Hasher::Out: Ord,
+		I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
+{
+	use trie::TrieConfiguration;
+	Ok(trie::trie_types::Layout::<Hasher>::trie_root(
+		build_pairs::<Header, I>(cht_size, cht_num, hashes)?
+	))
+}
+
+/// Build CHT-based header proof.
+pub fn build_proof<Header, Hasher, BlocksI, HashesI>(
+	cht_size: Header::Number,
+	cht_num: Header::Number,
+	blocks: BlocksI,
+	hashes: HashesI
+) -> ClientResult<StorageProof>
+	where
+		Header: HeaderT,
+		Hasher: hash_db::Hasher,
+		Hasher::Out: Ord,
+		BlocksI: IntoIterator<Item=Header::Number>,
+		HashesI: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
+{
+	let transaction = build_pairs::<Header, _>(cht_size, cht_num, hashes)?
+		.into_iter()
+		.map(|(k, v)| (None, k, Some(v)))
+		.collect::<Vec<_>>();
+	let mut storage = InMemoryState::<Hasher>::default().update(transaction);
+	let trie_storage = storage.as_trie_backend()
+		.expect("InMemoryState::as_trie_backend always returns Some; qed");
+	prove_read_on_trie_backend(
+		trie_storage,
+		blocks.into_iter().map(|number| encode_cht_key(number)),
+	).map_err(ClientError::Execution)
+}
+
+/// Check CHT-based header proof.
+pub fn check_proof<Header, Hasher>(
+	local_root: Header::Hash,
+	local_number: Header::Number,
+	remote_hash: Header::Hash,
+	remote_proof: StorageProof,
+) -> ClientResult<()>
+	where
+		Header: HeaderT,
+		Hasher: hash_db::Hasher,
+		Hasher::Out: Ord,
+{
+	do_check_proof::<Header, Hasher, _>(
+		local_root,
+		local_number,
+		remote_hash,
+		move |local_root, local_cht_key|
+			read_proof_check::<Hasher, _>(
+				local_root,
+				remote_proof,
+				::std::iter::once(local_cht_key),
+			)
+			.map(|mut map| map
+				.remove(local_cht_key)
+				.expect("checked proof of local_cht_key; qed"))
+			.map_err(|e| ClientError::from(e)),
+	)
+}
+
+/// Check CHT-based header proof on pre-created proving backend.
+pub fn check_proof_on_proving_backend<Header, Hasher>(
+	local_root: Header::Hash,
+	local_number: Header::Number,
+	remote_hash: Header::Hash,
+	proving_backend: &TrieBackend<MemoryDB<Hasher>, Hasher>,
+) -> ClientResult<()>
+	where
+		Header: HeaderT,
+		Hasher: hash_db::Hasher,
+		Hasher::Out: Ord,
+{
+	do_check_proof::<Header, Hasher, _>(
+		local_root,
+		local_number,
+		remote_hash,
+		|_, local_cht_key|
+			read_proof_check_on_proving_backend::<Hasher>(
+				proving_backend,
+				local_cht_key,
+			).map_err(|e| ClientError::from(e)),
+	)
+}
+
+/// Check CHT-based header proof using passed checker function.
+fn do_check_proof<Header, Hasher, F>(
+	local_root: Header::Hash,
+	local_number: Header::Number,
+	remote_hash: Header::Hash,
+	checker: F,
+) -> ClientResult<()>
+	where
+		Header: HeaderT,
+		Hasher: hash_db::Hasher,
+		Hasher::Out: Ord,
+		F: FnOnce(Hasher::Out, &[u8]) -> ClientResult<Option<Vec<u8>>>,
+{
+	let root: Hasher::Out = convert_hash(&local_root);
+	let local_cht_key = encode_cht_key(local_number);
+	let local_cht_value = checker(root, &local_cht_key)?;
+	let local_cht_value = local_cht_value.ok_or_else(|| ClientError::InvalidCHTProof)?;
+	let local_hash = decode_cht_value(&local_cht_value).ok_or_else(|| ClientError::InvalidCHTProof)?;
+	match &local_hash[..] == remote_hash.as_ref() {
+		true => Ok(()),
+		false => Err(ClientError::InvalidCHTProof.into()),
+	}
+
+}
+
+/// Group ordered blocks by CHT number and call functor with blocks of each group.
+pub fn for_each_cht_group<Header, I, F, P>(
+	cht_size: Header::Number,
+	blocks: I,
+	mut functor: F,
+	mut functor_param: P,
+) -> ClientResult<()>
+	where
+		Header: HeaderT,
+		I: IntoIterator<Item=Header::Number>,
+		F: FnMut(P, Header::Number, Vec<Header::Number>) -> ClientResult<P>,
+{
+	let mut current_cht_num = None;
+	let mut current_cht_blocks = Vec::new();
+	for block in blocks {
+		let new_cht_num = match block_to_cht_number(cht_size, block) {
+			Some(new_cht_num) => new_cht_num,
+			None => return Err(ClientError::Backend(format!(
+				"Cannot compute CHT root for the block #{}", block)).into()
+			),
+		};
+
+		let advance_to_next_cht = current_cht_num.is_some() && current_cht_num != Some(new_cht_num);
+		if advance_to_next_cht {
+			let current_cht_num = current_cht_num.expect("advance_to_next_cht is true;
+				it is true only when current_cht_num is Some; qed");
+			assert!(new_cht_num > current_cht_num, "for_each_cht_group only supports ordered iterators");
+
+			functor_param = functor(
+				functor_param,
+				current_cht_num,
+				::std::mem::replace(&mut current_cht_blocks, Vec::new()),
+			)?;
+		}
+
+		current_cht_blocks.push(block);
+		current_cht_num = Some(new_cht_num);
+	}
+
+	if let Some(current_cht_num) = current_cht_num {
+		functor(
+			functor_param,
+			current_cht_num,
+			::std::mem::replace(&mut current_cht_blocks, Vec::new()),
+		)?;
+	}
+
+	Ok(())
+}
+
+/// Build pairs for computing CHT.
+fn build_pairs<Header, I>(
+	cht_size: Header::Number,
+	cht_num: Header::Number,
+	hashes: I
+) -> ClientResult<Vec<(Vec<u8>, Vec<u8>)>>
+	where
+		Header: HeaderT,
+		I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
+{
+	let start_num = start_number(cht_size, cht_num);
+	let mut pairs = Vec::new();
+	let mut hash_index = Header::Number::zero();
+	for hash in hashes.into_iter() {
+		let hash = hash?.ok_or_else(|| ClientError::from(
+			ClientError::MissingHashRequiredForCHT
+		))?;
+		pairs.push((
+			encode_cht_key(start_num + hash_index).to_vec(),
+			encode_cht_value(hash)
+		));
+		hash_index += Header::Number::one();
+		if hash_index == cht_size {
+			break;
+		}
+	}
+
+	if hash_index == cht_size {
+		Ok(pairs)
+	} else {
+		Err(ClientError::MissingHashRequiredForCHT)
+	}
+}
+
+/// Get the starting block of a given CHT.
+/// CHT 0 includes block 1...SIZE,
+/// CHT 1 includes block SIZE + 1 ... 2*SIZE
+/// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE).
+/// This is because the genesis hash is assumed to be known
+/// and including it would be redundant.
+pub fn start_number<N: SimpleArithmetic>(cht_size: N, cht_num: N) -> N {
+	(cht_num * cht_size) + N::one()
+}
+
+/// Get the ending block of a given CHT.
+pub fn end_number<N: SimpleArithmetic>(cht_size: N, cht_num: N) -> N {
+	(cht_num + N::one()) * cht_size
+}
+
+/// Convert a block number to a CHT number.
+/// Returns `None` for `block_num` == 0, `Some` otherwise.
+pub fn block_to_cht_number<N: SimpleArithmetic>(cht_size: N, block_num: N) -> Option<N> {
+	if block_num == N::zero() {
+		None
+	} else {
+		Some((block_num - N::one()) / cht_size)
+	}
+}
+
+/// Convert header number into CHT key.
+pub fn encode_cht_key<N: Encode>(number: N) -> Vec<u8> {
+	number.encode()
+}
+
+/// Convert header hash into CHT value.
+fn encode_cht_value<Hash: AsRef<[u8]>>(hash: Hash) -> Vec<u8> {
+	hash.as_ref().to_vec()
+}
+
+/// Convert CHT value into block header hash.
+pub fn decode_cht_value(value: &[u8]) -> Option<H256> {
+	match value.len() {
+		32 => Some(H256::from_slice(&value[0..32])),
+		_ => None,
+	}
+
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::{Blake2Hasher};
+	use test_client::runtime::Header;
+	use super::*;
+
+	#[test]
+	fn is_build_required_works() {
+		assert_eq!(is_build_required(SIZE, 0u32.into()), None);
+		assert_eq!(is_build_required(SIZE, 1u32.into()), None);
+		assert_eq!(is_build_required(SIZE, SIZE), None);
+		assert_eq!(is_build_required(SIZE, SIZE + 1), None);
+		assert_eq!(is_build_required(SIZE, 2 * SIZE), None);
+		assert_eq!(is_build_required(SIZE, 2 * SIZE + 1), Some(0));
+		assert_eq!(is_build_required(SIZE, 3 * SIZE), None);
+		assert_eq!(is_build_required(SIZE, 3 * SIZE + 1), Some(1));
+	}
+
+	#[test]
+	fn start_number_works() {
+		assert_eq!(start_number(SIZE, 0u32), 1u32);
+		assert_eq!(start_number(SIZE, 1u32), SIZE + 1);
+		assert_eq!(start_number(SIZE, 2u32), SIZE + SIZE + 1);
+	}
+
+	#[test]
+	fn end_number_works() {
+		assert_eq!(end_number(SIZE, 0u32), SIZE);
+		assert_eq!(end_number(SIZE, 1u32), SIZE + SIZE);
+		assert_eq!(end_number(SIZE, 2u32), SIZE + SIZE + SIZE);
+	}
+
+	#[test]
+	fn build_pairs_fails_when_no_enough_blocks() {
+		assert!(build_pairs::<Header, _>(SIZE as _, 0,
+			::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2)).is_err());
+	}
+
+	#[test]
+	fn build_pairs_fails_when_missing_block() {
+		assert!(build_pairs::<Header, _>(
+			SIZE as _,
+			0,
+			::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
+				.take(SIZE as usize / 2)
+				.chain(::std::iter::once(Ok(None)))
+				.chain(::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2))))
+					.take(SIZE as usize / 2 - 1))
+		).is_err());
+	}
+
+	#[test]
+	fn compute_root_works() {
+		assert!(compute_root::<Header, Blake2Hasher, _>(
+			SIZE as _,
+			42,
+			::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
+				.take(SIZE as usize)
+		).is_ok());
+	}
+
+	#[test]
+	#[should_panic]
+	fn build_proof_panics_when_querying_wrong_block() {
+		assert!(build_proof::<Header, Blake2Hasher, _, _>(
+			SIZE as _,
+			0,
+			vec![(SIZE * 1000) as u64],
+			::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
+				.take(SIZE as usize)
+		).is_err());
+	}
+
+	#[test]
+	fn build_proof_works() {
+		assert!(build_proof::<Header, Blake2Hasher, _, _>(
+			SIZE as _,
+			0,
+			vec![(SIZE / 2) as u64],
+			::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1))))
+				.take(SIZE as usize)
+		).is_ok());
+	}
+
+	#[test]
+	#[should_panic]
+	fn for_each_cht_group_panics() {
+		let cht_size = SIZE as u64;
+		let _ = for_each_cht_group::<Header, _, _, _>(
+			cht_size,
+			vec![cht_size * 5, cht_size * 2],
+			|_, _, _| Ok(()),
+			(),
+		);
+	}
+
+	#[test]
+	fn for_each_cht_group_works() {
+		let cht_size = SIZE as u64;
+		let _ = for_each_cht_group::<Header, _, _, _>(
+			cht_size,
+			vec![
+				cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5,
+				cht_size * 4 + 1, cht_size * 4 + 7,
+				cht_size * 6 + 1
+			], |_, cht_num, blocks| {
+				match cht_num {
+					2 => assert_eq!(blocks, vec![cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5]),
+					4 => assert_eq!(blocks, vec![cht_size * 4 + 1, cht_size * 4 + 7]),
+					6 => assert_eq!(blocks, vec![cht_size * 6 + 1]),
+					_ => unreachable!(),
+				}
+
+				Ok(())
+			}, ()
+		);
+	}
+}

substrate/client/src/genesis.rs

@@ -0,0 +1,243 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Tool for creating the genesis block.
+
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Hash as HashT, Zero};
+
+/// Create a genesis block, given the initial storage.
+pub fn construct_genesis_block<
+	Block: BlockT
+> (
+	state_root: Block::Hash
+) -> Block {
+	let extrinsics_root = <<<Block as BlockT>::Header as HeaderT>::Hashing as HashT>::trie_root(
+		Vec::new(),
+	);
+
+	Block::new(
+		<<Block as BlockT>::Header as HeaderT>::new(
+			Zero::zero(),
+			extrinsics_root,
+			state_root,
+			Default::default(),
+			Default::default()
+		),
+		Default::default()
+	)
+}
+
+#[cfg(test)]
+mod tests {
+	use codec::{Encode, Decode, Joiner};
+	use executor::native_executor_instance;
+	use state_machine::{
+		StateMachine, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage,
+	};
+	use state_machine::backend::InMemory;
+	use test_client::{
+		runtime::genesismap::{GenesisConfig, insert_genesis_block},
+		runtime::{Hash, Transfer, Block, BlockNumber, Header, Digest},
+		AccountKeyring, Sr25519Keyring,
+	};
+	use primitives::{Blake2Hasher, map};
+	use hex_literal::*;
+
+	native_executor_instance!(
+		Executor,
+		test_client::runtime::api::dispatch,
+		test_client::runtime::native_version
+	);
+
+	fn executor() -> executor::NativeExecutor<Executor> {
+		executor::NativeExecutor::new(executor::WasmExecutionMethod::Interpreted, None)
+	}
+
+	fn construct_block(
+		backend: &InMemory<Blake2Hasher>,
+		number: BlockNumber,
+		parent_hash: Hash,
+		state_root: Hash,
+		txs: Vec<Transfer>
+	) -> (Vec<u8>, Hash) {
+		use trie::{TrieConfiguration, trie_types::Layout};
+
+		let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::<Vec<_>>();
+
+		let iter = transactions.iter().map(Encode::encode);
+		let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter).into();
+
+		let mut header = Header {
+			parent_hash,
+			number,
+			state_root,
+			extrinsics_root,
+			digest: Digest { logs: vec![], },
+		};
+		let hash = header.hash();
+		let mut overlay = OverlayedChanges::default();
+
+		StateMachine::new(
+			backend,
+			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+			None,
+			&mut overlay,
+			&executor(),
+			"Core_initialize_block",
+			&header.encode(),
+			None,
+		).execute(
+			ExecutionStrategy::NativeElseWasm,
+		).unwrap();
+
+		for tx in transactions.iter() {
+			StateMachine::new(
+				backend,
+				Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+				None,
+				&mut overlay,
+				&executor(),
+				"BlockBuilder_apply_extrinsic",
+				&tx.encode(),
+				None,
+			).execute(
+				ExecutionStrategy::NativeElseWasm,
+			).unwrap();
+		}
+
+		let (ret_data, _, _) = StateMachine::new(
+			backend,
+			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+			None,
+			&mut overlay,
+			&executor(),
+			"BlockBuilder_finalize_block",
+			&[],
+			None,
+		).execute(
+			ExecutionStrategy::NativeElseWasm,
+		).unwrap();
+		header = Header::decode(&mut &ret_data[..]).unwrap();
+
+		(vec![].and(&Block { header, extrinsics: transactions }), hash)
+	}
+
+	fn block1(genesis_hash: Hash, backend: &InMemory<Blake2Hasher>) -> (Vec<u8>, Hash) {
+		construct_block(
+			backend,
+			1,
+			genesis_hash,
+			hex!("25e5b37074063ab75c889326246640729b40d0c86932edc527bc80db0e04fe5c").into(),
+			vec![Transfer {
+				from: AccountKeyring::One.into(),
+				to: AccountKeyring::Two.into(),
+				amount: 69,
+				nonce: 0,
+			}]
+		)
+	}
+
+	#[test]
+	fn construct_genesis_should_work_with_native() {
+		let mut storage = GenesisConfig::new(false,
+			vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			1000,
+			None,
+			map![],
+			map![],
+		).genesis_map();
+		let genesis_hash = insert_genesis_block(&mut storage);
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let _ = StateMachine::new(
+			&backend,
+			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+			None,
+			&mut overlay,
+			&executor(),
+			"Core_execute_block",
+			&b1data,
+			None,
+		).execute(
+			ExecutionStrategy::NativeElseWasm,
+		).unwrap();
+	}
+
+	#[test]
+	fn construct_genesis_should_work_with_wasm() {
+		let mut storage = GenesisConfig::new(false,
+			vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			1000,
+			None,
+			map![],
+			map![],
+		).genesis_map();
+		let genesis_hash = insert_genesis_block(&mut storage);
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let _ = StateMachine::new(
+			&backend,
+			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+			None,
+			&mut overlay,
+			&executor(),
+			"Core_execute_block",
+			&b1data,
+			None,
+		).execute(
+			ExecutionStrategy::AlwaysWasm,
+		).unwrap();
+	}
+
+	#[test]
+	fn construct_genesis_with_bad_transaction_should_panic() {
+		let mut storage = GenesisConfig::new(false,
+			vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()],
+			vec![AccountKeyring::One.into(), AccountKeyring::Two.into()],
+			68,
+			None,
+			map![],
+			map![],
+		).genesis_map();
+		let genesis_hash = insert_genesis_block(&mut storage);
+
+		let backend = InMemory::from(storage);
+		let (b1data, _b1hash) = block1(genesis_hash, &backend);
+
+		let mut overlay = OverlayedChanges::default();
+		let r = StateMachine::new(
+			&backend,
+			Some(&InMemoryChangesTrieStorage::<_, u64>::new()),
+			None,
+			&mut overlay,
+			&executor(),
+			"Core_execute_block",
+			&b1data,
+			None,
+		).execute(
+			ExecutionStrategy::NativeElseWasm,
+		);
+		assert!(r.is_err());
+	}
+}

substrate/client/src/in_mem.rs

@@ -0,0 +1,855 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! In memory client backend
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+use parking_lot::{RwLock, Mutex};
+use primitives::{ChangesTrieConfiguration, storage::well_known_keys};
+use sr_primitives::generic::{BlockId, DigestItem};
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor};
+use sr_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay};
+use state_machine::backend::{Backend as StateBackend, InMemory};
+use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId, ChangesTrieTransaction};
+use hash_db::{Hasher, Prefix};
+use trie::MemoryDB;
+use header_metadata::{CachedHeaderMetadata, HeaderMetadata};
+
+use client_api::{
+	error,
+	backend::{self, NewBlockState, StorageCollection, ChildStorageCollection},
+	blockchain::{
+		self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId
+	},
+	offchain::{
+		InMemOffchainStorage as OffchainStorage
+	}
+};
+use crate::leaves::LeafSet;
+
+struct PendingBlock<B: BlockT> {
+	block: StoredBlock<B>,
+	state: NewBlockState,
+}
+
+#[derive(PartialEq, Eq, Clone)]
+enum StoredBlock<B: BlockT> {
+	Header(B::Header, Option<Justification>),
+	Full(B, Option<Justification>),
+}
+
+impl<B: BlockT> StoredBlock<B> {
+	fn new(header: B::Header, body: Option<Vec<B::Extrinsic>>, just: Option<Justification>) -> Self {
+		match body {
+			Some(body) => StoredBlock::Full(B::new(header, body), just),
+			None => StoredBlock::Header(header, just),
+		}
+	}
+
+	fn header(&self) -> &B::Header {
+		match *self {
+			StoredBlock::Header(ref h, _) => h,
+			StoredBlock::Full(ref b, _) => b.header(),
+		}
+	}
+
+	fn justification(&self) -> Option<&Justification> {
+		match *self {
+			StoredBlock::Header(_, ref j) | StoredBlock::Full(_, ref j) => j.as_ref()
+		}
+	}
+
+	fn extrinsics(&self) -> Option<&[B::Extrinsic]> {
+		match *self {
+			StoredBlock::Header(_, _) => None,
+			StoredBlock::Full(ref b, _) => Some(b.extrinsics()),
+		}
+	}
+
+	fn into_inner(self) -> (B::Header, Option<Vec<B::Extrinsic>>, Option<Justification>) {
+		match self {
+			StoredBlock::Header(header, just) => (header, None, just),
+			StoredBlock::Full(block, just) => {
+				let (header, body) = block.deconstruct();
+				(header, Some(body), just)
+			}
+		}
+	}
+}
+
+#[derive(Clone)]
+struct BlockchainStorage<Block: BlockT> {
+	blocks: HashMap<Block::Hash, StoredBlock<Block>>,
+	hashes: HashMap<NumberFor<Block>, Block::Hash>,
+	best_hash: Block::Hash,
+	best_number: NumberFor<Block>,
+	finalized_hash: Block::Hash,
+	finalized_number: NumberFor<Block>,
+	genesis_hash: Block::Hash,
+	header_cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
+	changes_trie_cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
+	leaves: LeafSet<Block::Hash, NumberFor<Block>>,
+	aux: HashMap<Vec<u8>, Vec<u8>>,
+}
+
+/// In-memory blockchain. Supports concurrent reads.
+pub struct Blockchain<Block: BlockT> {
+	storage: Arc<RwLock<BlockchainStorage<Block>>>,
+}
+
+impl<Block: BlockT + Clone> Clone for Blockchain<Block> {
+	fn clone(&self) -> Self {
+		let storage = Arc::new(RwLock::new(self.storage.read().clone()));
+		Blockchain {
+			storage: storage.clone(),
+		}
+	}
+}
+
+impl<Block: BlockT> Blockchain<Block> {
+	/// Get header hash of given block.
+	pub fn id(&self, id: BlockId<Block>) -> Option<Block::Hash> {
+		match id {
+			BlockId::Hash(h) => Some(h),
+			BlockId::Number(n) => self.storage.read().hashes.get(&n).cloned(),
+		}
+	}
+
+	/// Create new in-memory blockchain storage.
+	pub fn new() -> Blockchain<Block> {
+		let storage = Arc::new(RwLock::new(
+			BlockchainStorage {
+				blocks: HashMap::new(),
+				hashes: HashMap::new(),
+				best_hash: Default::default(),
+				best_number: Zero::zero(),
+				finalized_hash: Default::default(),
+				finalized_number: Zero::zero(),
+				genesis_hash: Default::default(),
+				header_cht_roots: HashMap::new(),
+				changes_trie_cht_roots: HashMap::new(),
+				leaves: LeafSet::new(),
+				aux: HashMap::new(),
+			}));
+		Blockchain {
+			storage: storage.clone(),
+		}
+	}
+
+	/// Insert a block header and associated data.
+	pub fn insert(
+		&self,
+		hash: Block::Hash,
+		header: <Block as BlockT>::Header,
+		justification: Option<Justification>,
+		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
+		new_state: NewBlockState,
+	) -> client_api::error::Result<()> {
+		let number = header.number().clone();
+		if new_state.is_best() {
+			self.apply_head(&header)?;
+		}
+
+		{
+			let mut storage = self.storage.write();
+			storage.leaves.import(hash.clone(), number.clone(), header.parent_hash().clone());
+			storage.blocks.insert(hash.clone(), StoredBlock::new(header, body, justification));
+
+			if let NewBlockState::Final = new_state {
+				storage.finalized_hash = hash;
+				storage.finalized_number = number.clone();
+			}
+
+			if number == Zero::zero() {
+				storage.genesis_hash = hash;
+			}
+		}
+
+		Ok(())
+	}
+
+	/// Get total number of blocks.
+	pub fn blocks_count(&self) -> usize {
+		self.storage.read().blocks.len()
+	}
+
+	/// Compare this blockchain with another in-mem blockchain
+	pub fn equals_to(&self, other: &Self) -> bool {
+		self.canon_equals_to(other) && self.storage.read().blocks == other.storage.read().blocks
+	}
+
+	/// Compare canonical chain to other canonical chain.
+	pub fn canon_equals_to(&self, other: &Self) -> bool {
+		let this = self.storage.read();
+		let other = other.storage.read();
+			this.hashes == other.hashes
+			&& this.best_hash == other.best_hash
+			&& this.best_number == other.best_number
+			&& this.genesis_hash == other.genesis_hash
+	}
+
+	/// Insert header CHT root.
+	pub fn insert_cht_root(&self, block: NumberFor<Block>, cht_root: Block::Hash) {
+		self.storage.write().header_cht_roots.insert(block, cht_root);
+	}
+
+	/// Set an existing block as head.
+	pub fn set_head(&self, id: BlockId<Block>) -> error::Result<()> {
+		let header = match self.header(id)? {
+			Some(h) => h,
+			None => return Err(error::Error::UnknownBlock(format!("{}", id))),
+		};
+
+		self.apply_head(&header)
+	}
+
+	fn apply_head(&self, header: &<Block as BlockT>::Header) -> error::Result<()> {
+		let hash = header.hash();
+		let number = header.number();
+
+		// Note: this may lock storage, so it must happen before obtaining storage
+		// write lock.
+		let best_tree_route = {
+			let best_hash = self.storage.read().best_hash;
+			if &best_hash == header.parent_hash() {
+				None
+			} else {
+				let route = header_metadata::tree_route(self, best_hash, *header.parent_hash())?;
+				Some(route)
+			}
+		};
+
+		let mut storage = self.storage.write();
+
+		if let Some(tree_route) = best_tree_route {
+			// apply retraction and enaction when reorganizing up to parent hash
+			let enacted = tree_route.enacted();
+
+			for entry in enacted {
+				storage.hashes.insert(entry.number, entry.hash);
+			}
+
+			for entry in tree_route.retracted().iter().skip(enacted.len()) {
+				storage.hashes.remove(&entry.number);
+			}
+		}
+
+		storage.best_hash = hash.clone();
+		storage.best_number = number.clone();
+		storage.hashes.insert(number.clone(), hash.clone());
+
+		Ok(())
+	}
+
+	fn finalize_header(&self, id: BlockId<Block>, justification: Option<Justification>) -> error::Result<()> {
+		let hash = match self.header(id)? {
+			Some(h) => h.hash(),
+			None => return Err(error::Error::UnknownBlock(format!("{}", id))),
+		};
+
+		let mut storage = self.storage.write();
+		storage.finalized_hash = hash;
+
+		if justification.is_some() {
+			let block = storage.blocks.get_mut(&hash)
+				.expect("hash was fetched from a block in the db; qed");
+
+			let block_justification = match block {
+				StoredBlock::Header(_, ref mut j) | StoredBlock::Full(_, ref mut j) => j
+			};
+
+			*block_justification = justification;
+		}
+
+		Ok(())
+	}
+
+	fn write_aux(&self, ops: Vec<(Vec<u8>, Option<Vec<u8>>)>) {
+		let mut storage = self.storage.write();
+		for (k, v) in ops {
+			match v {
+				Some(v) => storage.aux.insert(k, v),
+				None => storage.aux.remove(&k),
+			};
+		}
+	}
+}
+
+impl<Block: BlockT> HeaderBackend<Block> for Blockchain<Block> {
+	fn header(&self, id: BlockId<Block>) -> error::Result<Option<<Block as BlockT>::Header>> {
+		Ok(self.id(id).and_then(|hash| {
+			self.storage.read().blocks.get(&hash).map(|b| b.header().clone())
+		}))
+	}
+
+	fn info(&self) -> blockchain::Info<Block> {
+		let storage = self.storage.read();
+		blockchain::Info {
+			best_hash: storage.best_hash,
+			best_number: storage.best_number,
+			genesis_hash: storage.genesis_hash,
+			finalized_hash: storage.finalized_hash,
+			finalized_number: storage.finalized_number,
+		}
+	}
+
+	fn status(&self, id: BlockId<Block>) -> error::Result<BlockStatus> {
+		match self.id(id).map_or(false, |hash| self.storage.read().blocks.contains_key(&hash)) {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	fn number(&self, hash: Block::Hash) -> error::Result<Option<NumberFor<Block>>> {
+		Ok(self.storage.read().blocks.get(&hash).map(|b| *b.header().number()))
+	}
+
+	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> error::Result<Option<Block::Hash>> {
+		Ok(self.id(BlockId::Number(number)))
+	}
+}
+
+impl<Block: BlockT> HeaderMetadata<Block> for Blockchain<Block> {
+	type Error = error::Error;
+
+	fn header_metadata(&self, hash: Block::Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
+		self.header(BlockId::hash(hash))?.map(|header| CachedHeaderMetadata::from(&header))
+			.ok_or(error::Error::UnknownBlock(format!("header not found: {}", hash)))
+	}
+
+	fn insert_header_metadata(&self, _hash: Block::Hash, _metadata: CachedHeaderMetadata<Block>) {
+		// No need to implement.
+	}
+	fn remove_header_metadata(&self, _hash: Block::Hash) {
+		// No need to implement.
+	}
+}
+
+impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
+	fn body(&self, id: BlockId<Block>) -> error::Result<Option<Vec<<Block as BlockT>::Extrinsic>>> {
+		Ok(self.id(id).and_then(|hash| {
+			self.storage.read().blocks.get(&hash)
+				.and_then(|b| b.extrinsics().map(|x| x.to_vec()))
+		}))
+	}
+
+	fn justification(&self, id: BlockId<Block>) -> error::Result<Option<Justification>> {
+		Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b|
+			b.justification().map(|x| x.clone()))
+		))
+	}
+
+	fn last_finalized(&self) -> error::Result<Block::Hash> {
+		Ok(self.storage.read().finalized_hash.clone())
+	}
+
+	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
+		None
+	}
+
+	fn leaves(&self) -> error::Result<Vec<Block::Hash>> {
+		Ok(self.storage.read().leaves.hashes())
+	}
+
+	fn children(&self, _parent_hash: Block::Hash) -> error::Result<Vec<Block::Hash>> {
+		unimplemented!()
+	}
+}
+
+impl<Block: BlockT> blockchain::ProvideCache<Block> for Blockchain<Block> {
+	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
+		None
+	}
+}
+
+impl<Block: BlockT> backend::AuxStore for Blockchain<Block> {
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item=&'a &'b [u8]>,
+	>(&self, insert: I, delete: D) -> error::Result<()> {
+		let mut storage = self.storage.write();
+		for (k, v) in insert {
+			storage.aux.insert(k.to_vec(), v.to_vec());
+		}
+		for k in delete {
+			storage.aux.remove(*k);
+		}
+		Ok(())
+	}
+
+	fn get_aux(&self, key: &[u8]) -> error::Result<Option<Vec<u8>>> {
+		Ok(self.storage.read().aux.get(key).cloned())
+	}
+}
+
+impl<Block: BlockT> client_api::light::Storage<Block> for Blockchain<Block>
+	where
+		Block::Hash: From<[u8; 32]>,
+{
+	fn import_header(
+		&self,
+		header: Block::Header,
+		_cache: HashMap<CacheKeyId, Vec<u8>>,
+		state: NewBlockState,
+		aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	) -> error::Result<()> {
+		let hash = header.hash();
+		self.insert(hash, header, None, None, state)?;
+
+		self.write_aux(aux_ops);
+		Ok(())
+	}
+
+	fn set_head(&self, id: BlockId<Block>) -> error::Result<()> {
+		Blockchain::set_head(self, id)
+	}
+
+	fn last_finalized(&self) -> error::Result<Block::Hash> {
+		Ok(self.storage.read().finalized_hash.clone())
+	}
+
+	fn finalize_header(&self, id: BlockId<Block>) -> error::Result<()> {
+		Blockchain::finalize_header(self, id, None)
+	}
+
+	fn header_cht_root(
+		&self,
+		_cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> error::Result<Block::Hash> {
+		self.storage.read().header_cht_roots.get(&block).cloned()
+			.ok_or_else(|| error::Error::Backend(format!("Header CHT for block {} not exists", block)))
+	}
+
+	fn changes_trie_cht_root(
+		&self,
+		_cht_size: NumberFor<Block>,
+		block: NumberFor<Block>,
+	) -> error::Result<Block::Hash> {
+		self.storage.read().changes_trie_cht_roots.get(&block).cloned()
+			.ok_or_else(|| error::Error::Backend(format!("Changes trie CHT for block {} not exists", block)))
+	}
+
+	fn cache(&self) -> Option<Arc<dyn blockchain::Cache<Block>>> {
+		None
+	}
+}
+
+/// In-memory operation.
+pub struct BlockImportOperation<Block: BlockT, H: Hasher> {
+	pending_block: Option<PendingBlock<Block>>,
+	pending_cache: HashMap<CacheKeyId, Vec<u8>>,
+	old_state: InMemory<H>,
+	new_state: Option<InMemory<H>>,
+	changes_trie_update: Option<MemoryDB<H>>,
+	aux: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
+	set_head: Option<BlockId<Block>>,
+}
+
+impl<Block, H> backend::BlockImportOperation<Block, H> for BlockImportOperation<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+
+	H::Out: Ord,
+{
+	type State = InMemory<H>;
+
+	fn state(&self) -> error::Result<Option<&Self::State>> {
+		Ok(Some(&self.old_state))
+	}
+
+	fn set_block_data(
+		&mut self,
+		header: <Block as BlockT>::Header,
+		body: Option<Vec<<Block as BlockT>::Extrinsic>>,
+		justification: Option<Justification>,
+		state: NewBlockState,
+	) -> error::Result<()> {
+		assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
+		self.pending_block = Some(PendingBlock {
+			block: StoredBlock::new(header, body, justification),
+			state,
+		});
+		Ok(())
+	}
+
+	fn update_cache(&mut self, cache: HashMap<CacheKeyId, Vec<u8>>) {
+		self.pending_cache = cache;
+	}
+
+	fn update_db_storage(&mut self, update: <InMemory<H> as StateBackend<H>>::Transaction) -> error::Result<()> {
+		self.new_state = Some(self.old_state.update(update));
+		Ok(())
+	}
+
+	fn update_changes_trie(&mut self, update: ChangesTrieTransaction<H, NumberFor<Block>>) -> error::Result<()> {
+		self.changes_trie_update = Some(update.0);
+		Ok(())
+	}
+
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> error::Result<H::Out> {
+		check_genesis_storage(&top, &children)?;
+
+		let child_delta = children.into_iter()
+			.map(|(storage_key, child_overlay)|
+				(storage_key, child_overlay.into_iter().map(|(k, v)| (k, Some(v)))));
+
+		let (root, transaction) = self.old_state.full_storage_root(
+			top.into_iter().map(|(k, v)| (k, Some(v))),
+			child_delta
+		);
+
+		self.new_state = Some(InMemory::from(transaction));
+		Ok(root)
+	}
+
+	fn insert_aux<I>(&mut self, ops: I) -> error::Result<()>
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		self.aux.append(&mut ops.into_iter().collect());
+		Ok(())
+	}
+
+	fn update_storage(
+		&mut self,
+		_update: StorageCollection,
+		_child_update: ChildStorageCollection,
+	) -> error::Result<()> {
+		Ok(())
+	}
+
+	fn mark_finalized(&mut self, block: BlockId<Block>, justification: Option<Justification>) -> error::Result<()> {
+		self.finalized_blocks.push((block, justification));
+		Ok(())
+	}
+
+	fn mark_head(&mut self, block: BlockId<Block>) -> error::Result<()> {
+		assert!(self.pending_block.is_none(), "Only one set block per operation is allowed");
+		self.set_head = Some(block);
+		Ok(())
+	}
+}
+
+/// In-memory backend. Keeps all states and blocks in memory.
+///
+/// > **Warning**: Doesn't support all the features necessary for a proper database. Only use this
+/// > struct for testing purposes. Do **NOT** use in production.
+pub struct Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	states: RwLock<HashMap<Block::Hash, InMemory<H>>>,
+	changes_trie_storage: ChangesTrieStorage<Block, H>,
+	blockchain: Blockchain<Block>,
+	import_lock: Mutex<()>,
+}
+
+impl<Block, H> Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	/// Create a new instance of in-mem backend.
+	pub fn new() -> Backend<Block, H> {
+		Backend {
+			states: RwLock::new(HashMap::new()),
+			changes_trie_storage: ChangesTrieStorage(InMemoryChangesTrieStorage::new()),
+			blockchain: Blockchain::new(),
+			import_lock: Default::default(),
+		}
+	}
+}
+
+impl<Block, H> backend::AuxStore for Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item=&'a &'b [u8]>,
+	>(&self, insert: I, delete: D) -> error::Result<()> {
+		self.blockchain.insert_aux(insert, delete)
+	}
+
+	fn get_aux(&self, key: &[u8]) -> error::Result<Option<Vec<u8>>> {
+		self.blockchain.get_aux(key)
+	}
+}
+
+impl<Block, H> backend::Backend<Block, H> for Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	type BlockImportOperation = BlockImportOperation<Block, H>;
+	type Blockchain = Blockchain<Block>;
+	type State = InMemory<H>;
+	type ChangesTrieStorage = ChangesTrieStorage<Block, H>;
+	type OffchainStorage = OffchainStorage;
+
+	fn begin_operation(&self) -> error::Result<Self::BlockImportOperation> {
+		let old_state = self.state_at(BlockId::Hash(Default::default()))?;
+		Ok(BlockImportOperation {
+			pending_block: None,
+			pending_cache: Default::default(),
+			old_state,
+			new_state: None,
+			changes_trie_update: None,
+			aux: Default::default(),
+			finalized_blocks: Default::default(),
+			set_head: None,
+		})
+	}
+
+	fn begin_state_operation(&self, operation: &mut Self::BlockImportOperation, block: BlockId<Block>) -> error::Result<()> {
+		operation.old_state = self.state_at(block)?;
+		Ok(())
+	}
+
+	fn commit_operation(&self, operation: Self::BlockImportOperation) -> error::Result<()> {
+		if !operation.finalized_blocks.is_empty() {
+			for (block, justification) in operation.finalized_blocks {
+				self.blockchain.finalize_header(block, justification)?;
+			}
+		}
+
+		if let Some(pending_block) = operation.pending_block {
+			let old_state = &operation.old_state;
+			let (header, body, justification) = pending_block.block.into_inner();
+
+			let hash = header.hash();
+
+			self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
+
+			let maybe_changes_trie_root = header.digest().log(DigestItem::as_changes_trie_root).cloned();
+			if let Some(changes_trie_root) = maybe_changes_trie_root {
+				if let Some(changes_trie_update) = operation.changes_trie_update {
+					self.changes_trie_storage.0.insert(
+						*header.number(),
+						changes_trie_root,
+						changes_trie_update
+					);
+				}
+			}
+
+			self.blockchain.insert(hash, header, justification, body, pending_block.state)?;
+		}
+
+		if !operation.aux.is_empty() {
+			self.blockchain.write_aux(operation.aux);
+		}
+
+		if let Some(set_head) = operation.set_head {
+			self.blockchain.set_head(set_head)?;
+		}
+
+		Ok(())
+	}
+
+	fn finalize_block(&self, block: BlockId<Block>, justification: Option<Justification>) -> error::Result<()> {
+		self.blockchain.finalize_header(block, justification)
+	}
+
+	fn blockchain(&self) -> &Self::Blockchain {
+		&self.blockchain
+	}
+
+	fn used_state_cache_size(&self) -> Option<usize> {
+		None
+	}
+
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		Some(&self.changes_trie_storage)
+	}
+
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
+		None
+	}
+
+	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State> {
+		match block {
+			BlockId::Hash(h) if h == Default::default() => {
+				return Ok(Self::State::default());
+			},
+			_ => {},
+		}
+
+		match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) {
+			Some(state) => Ok(state),
+			None => Err(error::Error::UnknownBlock(format!("{}", block))),
+		}
+	}
+
+	fn revert(&self, _n: NumberFor<Block>) -> error::Result<NumberFor<Block>> {
+		Ok(Zero::zero())
+	}
+
+	fn get_import_lock(&self) -> &Mutex<()> {
+		&self.import_lock
+	}
+}
+
+impl<Block, H> backend::LocalBackend<Block, H> for Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{}
+
+impl<Block, H> backend::RemoteBackend<Block, H> for Backend<Block, H>
+where
+	Block: BlockT,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	fn is_local_state_available(&self, block: &BlockId<Block>) -> bool {
+		self.blockchain.expect_block_number_from_id(block)
+			.map(|num| num.is_zero())
+			.unwrap_or(false)
+	}
+
+	fn remote_blockchain(&self) -> Arc<dyn crate::light::blockchain::RemoteBlockchain<Block>> {
+		unimplemented!()
+	}
+}
+
+/// Prunable in-memory changes trie storage.
+pub struct ChangesTrieStorage<Block: BlockT, H: Hasher>(InMemoryChangesTrieStorage<H, NumberFor<Block>>);
+impl<Block: BlockT, H: Hasher> backend::PrunableStateChangesTrieStorage<Block, H> for ChangesTrieStorage<Block, H> {
+	fn oldest_changes_trie_block(
+		&self,
+		_config: &ChangesTrieConfiguration,
+		_best_finalized: NumberFor<Block>,
+	) -> NumberFor<Block> {
+		Zero::zero()
+	}
+}
+
+impl<Block, H> state_machine::ChangesTrieRootsStorage<H, NumberFor<Block>> for ChangesTrieStorage<Block, H>
+	where
+		Block: BlockT,
+		H: Hasher,
+{
+	fn build_anchor(
+		&self,
+		_hash: H::Out,
+	) -> Result<state_machine::ChangesTrieAnchorBlockId<H::Out, NumberFor<Block>>, String> {
+		Err("Dummy implementation".into())
+	}
+
+	fn root(
+		&self,
+		_anchor: &ChangesTrieAnchorBlockId<H::Out, NumberFor<Block>>,
+		_block: NumberFor<Block>,
+	) -> Result<Option<H::Out>, String> {
+		Err("Dummy implementation".into())
+	}
+}
+
+impl<Block, H> state_machine::ChangesTrieStorage<H, NumberFor<Block>> for ChangesTrieStorage<Block, H>
+	where
+		Block: BlockT,
+		H: Hasher,
+{
+	fn as_roots_storage(&self) -> &dyn state_machine::ChangesTrieRootsStorage<H, NumberFor<Block>> {
+		self
+	}
+
+	fn with_cached_changed_keys(
+		&self,
+		_root: &H::Out,
+		_functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
+	) -> bool {
+		false
+	}
+
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<state_machine::DBValue>, String> {
+		self.0.get(key, prefix)
+	}
+}
+
+/// Check that genesis storage is valid.
+pub fn check_genesis_storage(top: &StorageOverlay, children: &ChildrenStorageOverlay) -> error::Result<()> {
+	if top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) {
+		return Err(error::Error::GenesisInvalid.into());
+	}
+
+	if children.keys().any(|child_key| !well_known_keys::is_child_storage_key(&child_key)) {
+		return Err(error::Error::GenesisInvalid.into());
+	}
+
+	Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+	use client_api::offchain::{OffchainStorage, InMemOffchainStorage};
+	use std::sync::Arc;
+	use test_client;
+	use primitives::Blake2Hasher;
+
+	type TestBackend = test_client::client::in_mem::Backend<test_client::runtime::Block, Blake2Hasher>;
+
+	#[test]
+	fn test_leaves_with_complex_block_tree() {
+		let backend = Arc::new(TestBackend::new());
+
+		test_client::trait_tests::test_leaves_for_backend(backend);
+	}
+
+	#[test]
+	fn test_blockchain_query_by_number_gets_canonical() {
+		let backend = Arc::new(TestBackend::new());
+
+		test_client::trait_tests::test_blockchain_query_by_number_gets_canonical(backend);
+	}
+
+	#[test]
+	fn in_memory_offchain_storage() {
+
+		let mut storage = InMemOffchainStorage::default();
+		assert_eq!(storage.get(b"A", b"B"), None);
+		assert_eq!(storage.get(b"B", b"A"), None);
+
+		storage.set(b"A", b"B", b"C");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+		assert_eq!(storage.get(b"B", b"A"), None);
+
+		storage.compare_and_set(b"A", b"B", Some(b"X"), b"D");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"C".to_vec()));
+		storage.compare_and_set(b"A", b"B", Some(b"C"), b"D");
+		assert_eq!(storage.get(b"A", b"B"), Some(b"D".to_vec()));
+
+		assert!(!storage.compare_and_set(b"B", b"A", Some(b""), b"Y"));
+		assert!(storage.compare_and_set(b"B", b"A", None, b"X"));
+		assert_eq!(storage.get(b"B", b"A"), Some(b"X".to_vec()));
+	}
+}

substrate/client/src/leaves.rs

@@ -0,0 +1,356 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper for managing the set of available leaves in the chain for DB implementations.
+
+use std::collections::BTreeMap;
+use std::cmp::Reverse;
+use kvdb::{KeyValueDB, DBTransaction};
+use sr_primitives::traits::SimpleArithmetic;
+use codec::{Encode, Decode};
+use client_api::error;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct LeafSetItem<H, N> {
+	hash: H,
+	number: Reverse<N>,
+}
+
+/// A displaced leaf after import.
+#[must_use = "Displaced items from the leaf set must be handled."]
+pub struct ImportDisplaced<H, N> {
+	new_hash: H,
+	displaced: LeafSetItem<H, N>,
+}
+
+/// Displaced leaves after finalization.
+#[must_use = "Displaced items from the leaf set must be handled."]
+pub struct FinalizationDisplaced<H, N> {
+	leaves: BTreeMap<Reverse<N>, Vec<H>>,
+}
+
+impl<H, N: Ord> FinalizationDisplaced<H, N> {
+	/// Merge with another. This should only be used for displaced items that
+	/// are produced within one transaction of each other.
+	pub fn merge(&mut self, mut other: Self) {
+		// this will ignore keys that are in duplicate, however
+		// if these are actually produced correctly via the leaf-set within
+		// one transaction, then there will be no overlap in the keys.
+		self.leaves.append(&mut other.leaves);
+	}
+}
+
+/// list of leaf hashes ordered by number (descending).
+/// stored in memory for fast access.
+/// this allows very fast checking and modification of active leaves.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct LeafSet<H, N> {
+	storage: BTreeMap<Reverse<N>, Vec<H>>,
+	pending_added: Vec<LeafSetItem<H, N>>,
+	pending_removed: Vec<H>,
+}
+
+impl<H, N> LeafSet<H, N> where
+	H: Clone + PartialEq + Decode + Encode,
+	N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode,
+{
+	/// Construct a new, blank leaf set.
+	pub fn new() -> Self {
+		Self {
+			storage: BTreeMap::new(),
+			pending_added: Vec::new(),
+			pending_removed: Vec::new(),
+		}
+	}
+
+	/// Read the leaf list from the DB, using given prefix for keys.
+	pub fn read_from_db(db: &dyn KeyValueDB, column: Option<u32>, prefix: &[u8]) -> error::Result<Self> {
+		let mut storage = BTreeMap::new();
+
+		for (key, value) in db.iter_from_prefix(column, prefix) {
+			if !key.starts_with(prefix) { break }
+			let raw_hash = &mut &key[prefix.len()..];
+			let hash = match Decode::decode(raw_hash) {
+				Ok(hash) => hash,
+				Err(_) => return Err(error::Error::Backend("Error decoding hash".into())),
+			};
+			let number = match Decode::decode(&mut &value[..]) {
+				Ok(number) => number,
+				Err(_) => return Err(error::Error::Backend("Error decoding number".into())),
+			};
+			storage.entry(Reverse(number)).or_insert_with(Vec::new).push(hash);
+		}
+		Ok(Self {
+			storage,
+			pending_added: Vec::new(),
+			pending_removed: Vec::new(),
+		})
+	}
+
+	/// update the leaf list on import. returns a displaced leaf if there was one.
+	pub fn import(&mut self, hash: H, number: N, parent_hash: H) -> Option<ImportDisplaced<H, N>> {
+		// avoid underflow for genesis.
+		let displaced = if number != N::zero() {
+			let new_number = Reverse(number.clone() - N::one());
+			let was_displaced = self.remove_leaf(&new_number, &parent_hash);
+
+			if was_displaced {
+				self.pending_removed.push(parent_hash.clone());
+				Some(ImportDisplaced {
+					new_hash: hash.clone(),
+					displaced: LeafSetItem {
+						hash: parent_hash,
+						number: new_number,
+					},
+				})
+			} else {
+				None
+			}
+		} else {
+			None
+		};
+
+		self.insert_leaf(Reverse(number.clone()), hash.clone());
+		self.pending_added.push(LeafSetItem { hash, number: Reverse(number) });
+		displaced
+	}
+
+	/// Note a block height finalized, displacing all leaves with number less than the finalized block's.
+	///
+	/// Although it would be more technically correct to also prune out leaves at the
+	/// same number as the finalized block, but with different hashes, the current behavior
+	/// is simpler and our assumptions about how finalization works means that those leaves
+	/// will be pruned soon afterwards anyway.
+	pub fn finalize_height(&mut self, number: N) -> FinalizationDisplaced<H, N> {
+		let boundary = if number == N::zero() {
+			return FinalizationDisplaced { leaves: BTreeMap::new() };
+		} else {
+			number - N::one()
+		};
+
+		let below_boundary = self.storage.split_off(&Reverse(boundary));
+		self.pending_removed.extend(below_boundary.values().flat_map(|h| h.iter()).cloned());
+		FinalizationDisplaced {
+			leaves: below_boundary,
+		}
+	}
+
+	/// Undo all pending operations.
+	///
+	/// This returns an `Undo` struct, where any
+	/// `Displaced` objects that have returned by previous method calls
+	/// should be passed to via the appropriate methods. Otherwise,
+	/// the on-disk state may get out of sync with in-memory state.
+	pub fn undo(&mut self) -> Undo<H, N> {
+		Undo { inner: self }
+	}
+
+	/// currently since revert only affects the canonical chain
+	/// we assume that parent has no further children
+	/// and we add it as leaf again
+	pub fn revert(&mut self, hash: H, number: N, parent_hash: H) {
+		self.insert_leaf(Reverse(number.clone() - N::one()), parent_hash);
+		self.remove_leaf(&Reverse(number), &hash);
+	}
+
+	/// returns an iterator over all hashes in the leaf set
+	/// ordered by their block number descending.
+	pub fn hashes(&self) -> Vec<H> {
+		self.storage.iter().flat_map(|(_, hashes)| hashes.iter()).cloned().collect()
+	}
+
+	/// Write the leaf list to the database transaction.
+	pub fn prepare_transaction(&mut self, tx: &mut DBTransaction, column: Option<u32>, prefix: &[u8]) {
+		let mut buf = prefix.to_vec();
+		for LeafSetItem { hash, number } in self.pending_added.drain(..) {
+			hash.using_encoded(|s| buf.extend(s));
+			tx.put_vec(column, &buf[..], number.0.encode());
+			buf.truncate(prefix.len()); // reuse allocation.
+		}
+		for hash in self.pending_removed.drain(..) {
+			hash.using_encoded(|s| buf.extend(s));
+			tx.delete(column, &buf[..]);
+			buf.truncate(prefix.len()); // reuse allocation.
+		}
+	}
+
+	#[cfg(test)]
+	fn contains(&self, number: N, hash: H) -> bool {
+		self.storage.get(&Reverse(number)).map_or(false, |hashes| hashes.contains(&hash))
+	}
+
+	fn insert_leaf(&mut self, number: Reverse<N>, hash: H) {
+		self.storage.entry(number).or_insert_with(Vec::new).push(hash);
+	}
+
+	// returns true if this leaf was contained, false otherwise.
+	fn remove_leaf(&mut self, number: &Reverse<N>, hash: &H) -> bool {
+		let mut empty = false;
+		let removed = self.storage.get_mut(number).map_or(false, |leaves| {
+			let mut found = false;
+			leaves.retain(|h| if h == hash {
+				found = true;
+				false
+			} else {
+				true
+			});
+
+			if leaves.is_empty() { empty = true }
+
+			found
+		});
+
+		if removed && empty {
+			self.storage.remove(number);
+		}
+
+		removed
+	}
+}
+
+/// Helper for undoing operations.
+pub struct Undo<'a, H: 'a, N: 'a> {
+	inner: &'a mut LeafSet<H, N>,
+}
+
+impl<'a, H: 'a, N: 'a> Undo<'a, H, N> where
+	H: Clone + PartialEq + Decode + Encode,
+	N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode,
+{
+	/// Undo an imported block by providing the displaced leaf.
+	pub fn undo_import(&mut self, displaced: ImportDisplaced<H, N>) {
+		let new_number = Reverse(displaced.displaced.number.0.clone() + N::one());
+		self.inner.remove_leaf(&new_number, &displaced.new_hash);
+		self.inner.insert_leaf(new_number, displaced.displaced.hash);
+	}
+
+	/// Undo a finalization operation by providing the displaced leaves.
+	pub fn undo_finalization(&mut self, mut displaced: FinalizationDisplaced<H, N>) {
+		self.inner.storage.append(&mut displaced.leaves);
+	}
+}
+
+impl<'a, H: 'a, N: 'a> Drop for Undo<'a, H, N> {
+	fn drop(&mut self) {
+		self.inner.pending_added.clear();
+		self.inner.pending_removed.clear();
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn it_works() {
+		let mut set = LeafSet::new();
+		set.import(0u32, 0u32, 0u32);
+
+		set.import(1_1, 1, 0);
+		set.import(2_1, 2, 1_1);
+		set.import(3_1, 3, 2_1);
+
+		assert!(set.contains(3, 3_1));
+		assert!(!set.contains(2, 2_1));
+		assert!(!set.contains(1, 1_1));
+		assert!(!set.contains(0, 0));
+
+		set.import(2_2, 2, 1_1);
+
+		assert!(set.contains(3, 3_1));
+		assert!(set.contains(2, 2_2));
+	}
+
+	#[test]
+	fn flush_to_disk() {
+		const PREFIX: &[u8] = b"abcdefg";
+		let db = ::kvdb_memorydb::create(0);
+
+		let mut set = LeafSet::new();
+		set.import(0u32, 0u32, 0u32);
+
+		set.import(1_1, 1, 0);
+		set.import(2_1, 2, 1_1);
+		set.import(3_1, 3, 2_1);
+
+		let mut tx = DBTransaction::new();
+
+		set.prepare_transaction(&mut tx, None, PREFIX);
+		db.write(tx).unwrap();
+
+		let set2 = LeafSet::read_from_db(&db, None, PREFIX).unwrap();
+		assert_eq!(set, set2);
+	}
+
+	#[test]
+	fn two_leaves_same_height_can_be_included() {
+		let mut set = LeafSet::new();
+
+		set.import(1_1u32, 10u32,0u32);
+		set.import(1_2, 10, 0);
+
+		assert!(set.storage.contains_key(&Reverse(10)));
+		assert!(set.contains(10, 1_1));
+		assert!(set.contains(10, 1_2));
+		assert!(!set.contains(10, 1_3));
+	}
+
+	#[test]
+	fn finalization_consistent_with_disk() {
+		const PREFIX: &[u8] = b"prefix";
+		let db = ::kvdb_memorydb::create(0);
+
+		let mut set = LeafSet::new();
+		set.import(10_1u32, 10u32, 0u32);
+		set.import(11_1, 11, 10_2);
+		set.import(11_2, 11, 10_2);
+		set.import(12_1, 12, 11_123);
+
+		assert!(set.contains(10, 10_1));
+
+		let mut tx = DBTransaction::new();
+		set.prepare_transaction(&mut tx, None, PREFIX);
+		db.write(tx).unwrap();
+
+		let _ = set.finalize_height(11);
+		let mut tx = DBTransaction::new();
+		set.prepare_transaction(&mut tx, None, PREFIX);
+		db.write(tx).unwrap();
+
+		assert!(set.contains(11, 11_1));
+		assert!(set.contains(11, 11_2));
+		assert!(set.contains(12, 12_1));
+		assert!(!set.contains(10, 10_1));
+
+		let set2 = LeafSet::read_from_db(&db, None, PREFIX).unwrap();
+		assert_eq!(set, set2);
+	}
+
+	#[test]
+	fn undo_finalization() {
+		let mut set = LeafSet::new();
+		set.import(10_1u32, 10u32, 0u32);
+		set.import(11_1, 11, 10_2);
+		set.import(11_2, 11, 10_2);
+		set.import(12_1, 12, 11_123);
+
+		let displaced = set.finalize_height(11);
+		assert!(!set.contains(10, 10_1));
+
+		set.undo().undo_finalization(displaced);
+		assert!(set.contains(10, 10_1));
+	}
+}

substrate/client/src/lib.rs

@@ -0,0 +1,105 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Substrate Client and associated logic.
+//!
+//! The [`Client`] is one of the most important components of Substrate. It mainly comprises two
+//! parts:
+//!
+//! - A database containing the blocks and chain state, generally referred to as
+//! the [`Backend`](backend::Backend).
+//! - A runtime environment, generally referred to as the [`Executor`](CallExecutor).
+//!
+//! # Initialization
+//!
+//! Creating a [`Client`] is done by calling the `new` method and passing to it a
+//! [`Backend`](backend::Backend) and an [`Executor`](CallExecutor).
+//!
+//! The former is typically provided by the `substrate-client-db` crate.
+//!
+//! The latter typically requires passing one of:
+//!
+//! - A [`LocalCallExecutor`] running the runtime locally.
+//! - A [`RemoteCallExecutor`](light::call_executor::RemoteCallExecutor) that will ask a
+//! third-party to perform the executions.
+//! - A [`RemoteOrLocalCallExecutor`](light::call_executor::RemoteOrLocalCallExecutor), combination
+//! of the two.
+//!
+//! Additionally, the fourth generic parameter of the `Client` is a marker type representing
+//! the ways in which the runtime can interface with the outside. Any code that builds a `Client`
+//! is responsible for putting the right marker.
+//!
+//! ## Example
+//!
+//! ```
+//! use std::sync::Arc;
+//! use substrate_client::{Client, in_mem::Backend, LocalCallExecutor};
+//! use primitives::Blake2Hasher;
+//! use sr_primitives::{StorageOverlay, ChildrenStorageOverlay};
+//! use executor::{NativeExecutor, WasmExecutionMethod};
+//!
+//! // In this example, we're using the `Block` and `RuntimeApi` types from the
+//! // `substrate-test-runtime-client` crate. These types are automatically generated when
+//! // compiling a runtime. In a typical use-case, these types would have been to be generated
+//! // from your runtime.
+//! use test_client::{LocalExecutor, runtime::Block, runtime::RuntimeApi};
+//!
+//! let backend = Arc::new(Backend::<Block, Blake2Hasher>::new());
+//! let client = Client::<_, _, _, RuntimeApi>::new(
+//! 	backend.clone(),
+//! 	LocalCallExecutor::new(
+//! 		backend.clone(),
+//! 		NativeExecutor::<LocalExecutor>::new(WasmExecutionMethod::Interpreted, None),
+//! 		None,
+//!		),
+//! 	// This parameter provides the storage for the chain genesis.
+//! 	<(StorageOverlay, ChildrenStorageOverlay)>::default(),
+//! 	Default::default(),
+//! 	Default::default(),
+//! );
+//! ```
+//!
+
+#![warn(missing_docs)]
+#![recursion_limit="128"]
+
+pub mod cht;
+pub mod in_mem;
+pub mod genesis;
+pub mod light;
+pub mod leaves;
+mod call_executor;
+mod client;
+
+pub use client_api::{
+	error,
+	blockchain,
+	blockchain::well_known_cache_keys,
+	blockchain::Info as ChainInfo,
+	notifications::{StorageEventStream, StorageChangeSet},
+	call_executor::CallExecutor,
+	utils,
+};
+pub use crate::call_executor::LocalCallExecutor;
+pub use crate::client::{
+	new_with_backend,
+	new_in_mem,
+	BlockBody, ImportNotifications, FinalityNotifications, BlockchainEvents,
+	BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification,
+	LongestChain, BlockOf, ProvideUncles, ForkBlocks, apply_aux,
+};
+pub use state_machine::{ExecutionStrategy, StorageProof};
+pub use crate::leaves::LeafSet;

substrate/client/src/light/backend.rs

@@ -0,0 +1,493 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client backend. Only stores headers and justifications of blocks.
+//! Everything else is requested from full nodes on demand.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use parking_lot::{RwLock, Mutex};
+
+use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay};
+use state_machine::{Backend as StateBackend, TrieBackend, backend::InMemory as InMemoryState, ChangesTrieTransaction};
+use sr_primitives::traits::{Block as BlockT, NumberFor, Zero, Header};
+use crate::in_mem::{self, check_genesis_storage};
+use client_api::{
+	backend::{
+		AuxStore, Backend as ClientBackend, BlockImportOperation, RemoteBackend, NewBlockState,
+		StorageCollection, ChildStorageCollection,
+	},
+	blockchain::{
+		HeaderBackend as BlockchainHeaderBackend, well_known_cache_keys,
+	},
+	error::{
+		Error as ClientError, Result as ClientResult
+	},
+	light::Storage as BlockchainStorage,
+	InMemOffchainStorage,
+};
+use crate::light::blockchain::{Blockchain};
+use hash_db::Hasher;
+use trie::MemoryDB;
+
+const IN_MEMORY_EXPECT_PROOF: &str = "InMemory state backend has Void error type and always succeeds; qed";
+
+/// Light client backend.
+pub struct Backend<S, H: Hasher> {
+	blockchain: Arc<Blockchain<S>>,
+	genesis_state: RwLock<Option<InMemoryState<H>>>,
+	import_lock: Mutex<()>,
+}
+
+/// Light block (header and justification) import operation.
+pub struct ImportOperation<Block: BlockT, S, H: Hasher> {
+	header: Option<Block::Header>,
+	cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+	leaf_state: NewBlockState,
+	aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	finalized_blocks: Vec<BlockId<Block>>,
+	set_head: Option<BlockId<Block>>,
+	storage_update: Option<InMemoryState<H>>,
+	_phantom: ::std::marker::PhantomData<(S)>,
+}
+
+/// Either in-memory genesis state, or locally-unavailable state.
+pub enum GenesisOrUnavailableState<H: Hasher> {
+	/// Genesis state - storage values are stored in-memory.
+	Genesis(InMemoryState<H>),
+	/// We know that state exists, but all calls will fail with error, because it
+	/// isn't locally available.
+	Unavailable,
+}
+
+impl<S, H: Hasher> Backend<S, H> {
+	/// Create new light backend.
+	pub fn new(blockchain: Arc<Blockchain<S>>) -> Self {
+		Self {
+			blockchain,
+			genesis_state: RwLock::new(None),
+			import_lock: Default::default(),
+		}
+	}
+
+	/// Get shared blockchain reference.
+	pub fn blockchain(&self) -> &Arc<Blockchain<S>> {
+		&self.blockchain
+	}
+}
+
+impl<S: AuxStore, H: Hasher> AuxStore for Backend<S, H> {
+	fn insert_aux<
+		'a,
+		'b: 'a,
+		'c: 'a,
+		I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+		D: IntoIterator<Item=&'a &'b [u8]>,
+	>(&self, insert: I, delete: D) -> ClientResult<()> {
+		self.blockchain.storage().insert_aux(insert, delete)
+	}
+
+	fn get_aux(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		self.blockchain.storage().get_aux(key)
+	}
+}
+
+impl<S, Block, H> ClientBackend<Block, H> for Backend<S, H> where
+	Block: BlockT,
+	S: BlockchainStorage<Block>,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	type BlockImportOperation = ImportOperation<Block, S, H>;
+	type Blockchain = Blockchain<S>;
+	type State = GenesisOrUnavailableState<H>;
+	type ChangesTrieStorage = in_mem::ChangesTrieStorage<Block, H>;
+	type OffchainStorage = InMemOffchainStorage;
+
+	fn begin_operation(&self) -> ClientResult<Self::BlockImportOperation> {
+		Ok(ImportOperation {
+			header: None,
+			cache: Default::default(),
+			leaf_state: NewBlockState::Normal,
+			aux_ops: Vec::new(),
+			finalized_blocks: Vec::new(),
+			set_head: None,
+			storage_update: None,
+			_phantom: Default::default(),
+		})
+	}
+
+	fn begin_state_operation(
+		&self,
+		_operation: &mut Self::BlockImportOperation,
+		_block: BlockId<Block>
+	) -> ClientResult<()> {
+		Ok(())
+	}
+
+	fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> ClientResult<()> {
+		if !operation.finalized_blocks.is_empty() {
+			for block in operation.finalized_blocks {
+				self.blockchain.storage().finalize_header(block)?;
+			}
+		}
+
+		if let Some(header) = operation.header {
+			let is_genesis_import = header.number().is_zero();
+			self.blockchain.storage().import_header(
+				header,
+				operation.cache,
+				operation.leaf_state,
+				operation.aux_ops,
+			)?;
+
+			// when importing genesis block => remember its state
+			if is_genesis_import {
+				*self.genesis_state.write() = operation.storage_update.take();
+			}
+		} else {
+			for (key, maybe_val) in operation.aux_ops {
+				match maybe_val {
+					Some(val) => self.blockchain.storage().insert_aux(
+						&[(&key[..], &val[..])],
+						::std::iter::empty(),
+					)?,
+					None => self.blockchain.storage().insert_aux(::std::iter::empty(), &[&key[..]])?,
+				}
+			}
+		}
+
+		if let Some(set_head) = operation.set_head {
+			self.blockchain.storage().set_head(set_head)?;
+		}
+
+		Ok(())
+	}
+
+	fn finalize_block(&self, block: BlockId<Block>, _justification: Option<Justification>) -> ClientResult<()> {
+		self.blockchain.storage().finalize_header(block)
+	}
+
+	fn blockchain(&self) -> &Blockchain<S> {
+		&self.blockchain
+	}
+
+	fn used_state_cache_size(&self) -> Option<usize> {
+		None
+	}
+
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		None
+	}
+
+	fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
+		None
+	}
+
+	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
+		let block_number = self.blockchain.expect_block_number_from_id(&block)?;
+
+		// special case for genesis block
+		if block_number.is_zero() {
+			if let Some(genesis_state) = self.genesis_state.read().clone() {
+				return Ok(GenesisOrUnavailableState::Genesis(genesis_state));
+			}
+		}
+
+		// else return unavailable state. We do not return error here, because error
+		// would mean that we do not know this state at all. But we know that it exists
+		Ok(GenesisOrUnavailableState::Unavailable)
+	}
+
+	fn revert(&self, _n: NumberFor<Block>) -> ClientResult<NumberFor<Block>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn get_import_lock(&self) -> &Mutex<()> {
+		&self.import_lock
+	}
+}
+
+impl<S, Block, H> RemoteBackend<Block, H> for Backend<S, H>
+where
+	Block: BlockT,
+	S: BlockchainStorage<Block> + 'static,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	fn is_local_state_available(&self, block: &BlockId<Block>) -> bool {
+		self.genesis_state.read().is_some()
+			&& self.blockchain.expect_block_number_from_id(block)
+				.map(|num| num.is_zero())
+				.unwrap_or(false)
+	}
+
+	fn remote_blockchain(&self) -> Arc<dyn crate::light::blockchain::RemoteBlockchain<Block>> {
+		self.blockchain.clone()
+	}
+}
+
+impl<S, Block, H> BlockImportOperation<Block, H> for ImportOperation<Block, S, H>
+where
+	Block: BlockT,
+	S: BlockchainStorage<Block>,
+	H: Hasher<Out=Block::Hash>,
+	H::Out: Ord,
+{
+	type State = GenesisOrUnavailableState<H>;
+
+	fn state(&self) -> ClientResult<Option<&Self::State>> {
+		// None means 'locally-stateless' backend
+		Ok(None)
+	}
+
+	fn set_block_data(
+		&mut self,
+		header: Block::Header,
+		_body: Option<Vec<Block::Extrinsic>>,
+		_justification: Option<Justification>,
+		state: NewBlockState,
+	) -> ClientResult<()> {
+		self.leaf_state = state;
+		self.header = Some(header);
+		Ok(())
+	}
+
+	fn update_cache(&mut self, cache: HashMap<well_known_cache_keys::Id, Vec<u8>>) {
+		self.cache = cache;
+	}
+
+	fn update_db_storage(&mut self, _update: <Self::State as StateBackend<H>>::Transaction) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
+	fn update_changes_trie(&mut self, _update: ChangesTrieTransaction<H, NumberFor<Block>>) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
+	fn reset_storage(&mut self, top: StorageOverlay, children: ChildrenStorageOverlay) -> ClientResult<H::Out> {
+		check_genesis_storage(&top, &children)?;
+
+		// this is only called when genesis block is imported => shouldn't be performance bottleneck
+		let mut storage: HashMap<Option<Vec<u8>>, StorageOverlay> = HashMap::new();
+		storage.insert(None, top);
+
+		// create a list of children keys to re-compute roots for
+		let child_delta = children.keys()
+			.cloned()
+			.map(|storage_key| (storage_key, None))
+			.collect::<Vec<_>>();
+
+		// make sure to persist the child storage
+		for (child_key, child_storage) in children {
+			storage.insert(Some(child_key), child_storage);
+		}
+
+		let storage_update: InMemoryState<H> = storage.into();
+		let (storage_root, _) = storage_update.full_storage_root(::std::iter::empty(), child_delta);
+		self.storage_update = Some(storage_update);
+
+		Ok(storage_root)
+	}
+
+	fn insert_aux<I>(&mut self, ops: I) -> ClientResult<()>
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		self.aux_ops.append(&mut ops.into_iter().collect());
+		Ok(())
+	}
+
+	fn update_storage(
+		&mut self,
+		_update: StorageCollection,
+		_child_update: ChildStorageCollection,
+	) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
+	fn mark_finalized(&mut self, block: BlockId<Block>, _justification: Option<Justification>) -> ClientResult<()> {
+		self.finalized_blocks.push(block);
+		Ok(())
+	}
+
+	fn mark_head(&mut self, block: BlockId<Block>) -> ClientResult<()> {
+		self.set_head = Some(block);
+		Ok(())
+	}
+}
+
+impl<H: Hasher> std::fmt::Debug for GenesisOrUnavailableState<H> {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.fmt(f),
+			GenesisOrUnavailableState::Unavailable => write!(f, "Unavailable"),
+		}
+	}
+}
+
+impl<H: Hasher> StateBackend<H> for GenesisOrUnavailableState<H>
+	where
+		H::Out: Ord,
+{
+	type Error = ClientError;
+	type Transaction = ();
+	type TrieBackendStorage = MemoryDB<H>;
+
+	fn storage(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) =>
+				Ok(state.storage(key).expect(IN_MEMORY_EXPECT_PROOF)),
+			GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient),
+		}
+	}
+
+	fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) =>
+				Ok(state.child_storage(storage_key, key).expect(IN_MEMORY_EXPECT_PROOF)),
+			GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient),
+		}
+	}
+
+	fn for_keys_with_prefix<A: FnMut(&[u8])>(&self, prefix: &[u8], action: A) {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.for_keys_with_prefix(prefix, action),
+			GenesisOrUnavailableState::Unavailable => (),
+		}
+	}
+
+	fn for_key_values_with_prefix<A: FnMut(&[u8], &[u8])>(&self, prefix: &[u8], action: A) {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.for_key_values_with_prefix(prefix, action),
+			GenesisOrUnavailableState::Unavailable => (),
+		}
+	}
+
+
+	fn for_keys_in_child_storage<A: FnMut(&[u8])>(&self, storage_key: &[u8], action: A) {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.for_keys_in_child_storage(storage_key, action),
+			GenesisOrUnavailableState::Unavailable => (),
+		}
+	}
+
+	fn for_child_keys_with_prefix<A: FnMut(&[u8])>(
+		&self,
+		storage_key: &[u8],
+		prefix: &[u8],
+		action: A,
+	) {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) =>
+				state.for_child_keys_with_prefix(storage_key, prefix, action),
+			GenesisOrUnavailableState::Unavailable => (),
+		}
+	}
+
+	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) =>
+				(state.storage_root(delta).0, ()),
+			GenesisOrUnavailableState::Unavailable => (H::Out::default(), ()),
+		}
+	}
+
+	fn child_storage_root<I>(&self, key: &[u8], delta: I) -> (Vec<u8>, bool, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => {
+				let (root, is_equal, _) = state.child_storage_root(key, delta);
+				(root, is_equal, ())
+			},
+			GenesisOrUnavailableState::Unavailable => (H::Out::default().as_ref().to_vec(), true, ()),
+		}
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.pairs(),
+			GenesisOrUnavailableState::Unavailable => Vec::new(),
+		}
+	}
+
+	fn keys(&self, prefix: &[u8]) -> Vec<Vec<u8>> {
+		match *self {
+			GenesisOrUnavailableState::Genesis(ref state) => state.keys(prefix),
+			GenesisOrUnavailableState::Unavailable => Vec::new(),
+		}
+	}
+
+	fn as_trie_backend(&mut self) -> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
+		match self {
+			GenesisOrUnavailableState::Genesis(ref mut state) => state.as_trie_backend(),
+			GenesisOrUnavailableState::Unavailable => None,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::Blake2Hasher;
+	use test_client::{self, runtime::Block};
+	use client_api::backend::NewBlockState;
+	use crate::light::blockchain::tests::{DummyBlockchain, DummyStorage};
+	use super::*;
+
+	#[test]
+	fn local_state_is_created_when_genesis_state_is_available() {
+		let def = Default::default();
+		let header0 = test_client::runtime::Header::new(0, def, def, def, Default::default());
+
+		let backend: Backend<_, Blake2Hasher> = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())));
+		let mut op = backend.begin_operation().unwrap();
+		op.set_block_data(header0, None, None, NewBlockState::Final).unwrap();
+		op.reset_storage(Default::default(), Default::default()).unwrap();
+		backend.commit_operation(op).unwrap();
+
+		match backend.state_at(BlockId::Number(0)).unwrap() {
+			GenesisOrUnavailableState::Genesis(_) => (),
+			_ => panic!("unexpected state"),
+		}
+	}
+
+	#[test]
+	fn unavailable_state_is_created_when_genesis_state_is_unavailable() {
+		let backend: Backend<_, Blake2Hasher> = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())));
+
+		match backend.state_at(BlockId::Number(0)).unwrap() {
+			GenesisOrUnavailableState::Unavailable => (),
+			_ => panic!("unexpected state"),
+		}
+	}
+
+	#[test]
+	fn light_aux_store_is_updated_via_non_importing_op() {
+		let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new())));
+		let mut op = ClientBackend::<Block, Blake2Hasher>::begin_operation(&backend).unwrap();
+		BlockImportOperation::<Block, Blake2Hasher>::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap();
+		ClientBackend::<Block, Blake2Hasher>::commit_operation(&backend, op).unwrap();
+
+		assert_eq!(AuxStore::get_aux(&backend, &[1]).unwrap(), Some(vec![2]));
+	}
+}

substrate/client/src/light/blockchain.rs

@@ -0,0 +1,318 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client blockchain backend. Only stores headers and justifications of recent
+//! blocks. CHT roots are stored for headers of ancient blocks.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use sr_primitives::{Justification, generic::BlockId};
+use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
+
+use header_metadata::{HeaderMetadata, CachedHeaderMetadata};
+pub use client_api::{
+	backend::{
+		AuxStore, NewBlockState
+	},
+	blockchain::{
+		Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache,
+		HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo, ProvideCache,
+		well_known_cache_keys,
+	},
+	error::{
+		Error as ClientError, Result as ClientResult
+	},
+	light::{
+		RemoteBlockchain, LocalOrRemote, Storage
+	}
+};
+use crate::cht;
+use crate::light::fetcher::{Fetcher, RemoteHeaderRequest};
+
+/// Light client blockchain.
+pub struct Blockchain<S> {
+	storage: S,
+}
+
+impl<S> Blockchain<S> {
+	/// Create new light blockchain backed with given storage.
+	pub fn new(storage: S) -> Self {
+		Self {
+			storage,
+		}
+	}
+
+	/// Get storage reference.
+	pub fn storage(&self) -> &S {
+		&self.storage
+	}
+}
+
+impl<S, Block> BlockchainHeaderBackend<Block> for Blockchain<S> where Block: BlockT, S: Storage<Block> {
+	fn header(&self, id: BlockId<Block>) -> ClientResult<Option<Block::Header>> {
+		match RemoteBlockchain::header(self, id)? {
+			LocalOrRemote::Local(header) => Ok(Some(header)),
+			LocalOrRemote::Remote(_) => Err(ClientError::NotAvailableOnLightClient),
+			LocalOrRemote::Unknown => Ok(None),
+		}
+	}
+
+	fn info(&self) -> BlockchainInfo<Block> {
+		self.storage.info()
+	}
+
+	fn status(&self, id: BlockId<Block>) -> ClientResult<BlockStatus> {
+		self.storage.status(id)
+	}
+
+	fn number(&self, hash: Block::Hash) -> ClientResult<Option<NumberFor<Block>>> {
+		self.storage.number(hash)
+	}
+
+	fn hash(&self, number: <<Block as BlockT>::Header as HeaderT>::Number) -> ClientResult<Option<Block::Hash>> {
+		self.storage.hash(number)
+	}
+}
+
+impl<S, Block> HeaderMetadata<Block> for Blockchain<S> where Block: BlockT, S: Storage<Block> {
+	type Error = ClientError;
+
+	fn header_metadata(&self, hash: Block::Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
+		self.storage.header_metadata(hash)
+	}
+
+	fn insert_header_metadata(&self, hash: Block::Hash, metadata: CachedHeaderMetadata<Block>) {
+		self.storage.insert_header_metadata(hash, metadata)
+	}
+
+	fn remove_header_metadata(&self, hash: Block::Hash) {
+		self.storage.remove_header_metadata(hash)
+	}
+}
+
+impl<S, Block> BlockchainBackend<Block> for Blockchain<S> where Block: BlockT, S: Storage<Block> {
+	fn body(&self, _id: BlockId<Block>) -> ClientResult<Option<Vec<Block::Extrinsic>>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn justification(&self, _id: BlockId<Block>) -> ClientResult<Option<Justification>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn last_finalized(&self) -> ClientResult<Block::Hash> {
+		self.storage.last_finalized()
+	}
+
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
+		self.storage.cache()
+	}
+
+	fn leaves(&self) -> ClientResult<Vec<Block::Hash>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn children(&self, _parent_hash: Block::Hash) -> ClientResult<Vec<Block::Hash>> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+}
+
+impl<S: Storage<Block>, Block: BlockT> ProvideCache<Block> for Blockchain<S> {
+	fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
+		self.storage.cache()
+	}
+}
+
+impl<S, Block: BlockT> RemoteBlockchain<Block> for Blockchain<S>
+	where
+		S: Storage<Block>,
+{
+	fn header(&self, id: BlockId<Block>) -> ClientResult<LocalOrRemote<
+		Block::Header,
+		RemoteHeaderRequest<Block::Header>,
+	>> {
+		// first, try to read header from local storage
+		if let Some(local_header) = self.storage.header(id)? {
+			return Ok(LocalOrRemote::Local(local_header));
+		}
+
+		// we need to know block number to check if it's a part of CHT
+		let number = match id {
+			BlockId::Hash(hash) => match self.storage.number(hash)? {
+				Some(number) => number,
+				None => return Ok(LocalOrRemote::Unknown),
+			},
+			BlockId::Number(number) => number,
+		};
+
+		// if the header is genesis (never pruned), non-canonical, or from future => return
+		if number.is_zero() || self.storage.status(BlockId::Number(number))? == BlockStatus::Unknown {
+			return Ok(LocalOrRemote::Unknown);
+		}
+
+		Ok(LocalOrRemote::Remote(RemoteHeaderRequest {
+			cht_root: self.storage.header_cht_root(cht::size(), number)?,
+			block: number,
+			retry_count: None,
+		}))
+	}
+}
+
+/// Returns future that resolves header either locally, or remotely.
+pub fn future_header<Block: BlockT, F: Fetcher<Block>>(
+	blockchain: &dyn RemoteBlockchain<Block>,
+	fetcher: &F,
+	id: BlockId<Block>,
+) -> impl Future<Output = Result<Option<Block::Header>, ClientError>> {
+	use futures03::future::{ready, Either, FutureExt};
+
+	match blockchain.header(id) {
+		Ok(LocalOrRemote::Remote(request)) => Either::Left(
+			fetcher
+				.remote_header(request)
+				.then(|header| ready(header.map(Some)))
+		),
+		Ok(LocalOrRemote::Unknown) => Either::Right(ready(Ok(None))),
+		Ok(LocalOrRemote::Local(local_header)) => Either::Right(ready(Ok(Some(local_header)))),
+		Err(err) => Either::Right(ready(Err(err))),
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use std::collections::HashMap;
+	use parking_lot::Mutex;
+	use test_client::runtime::{Hash, Block, Header};
+	use client_api::blockchain::Info;
+	use super::*;
+
+	pub type DummyBlockchain = Blockchain<DummyStorage>;
+
+	pub struct DummyStorage {
+		pub changes_tries_cht_roots: HashMap<u64, Hash>,
+		pub aux_store: Mutex<HashMap<Vec<u8>, Vec<u8>>>,
+	}
+
+	impl DummyStorage {
+		pub fn new() -> Self {
+			DummyStorage {
+				changes_tries_cht_roots: HashMap::new(),
+				aux_store: Mutex::new(HashMap::new()),
+			}
+		}
+	}
+
+	impl BlockchainHeaderBackend<Block> for DummyStorage {
+		fn header(&self, _id: BlockId<Block>) -> ClientResult<Option<Header>> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn info(&self) -> Info<Block> {
+			panic!("Test error")
+		}
+
+		fn status(&self, _id: BlockId<Block>) -> ClientResult<BlockStatus> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn number(&self, hash: Hash) -> ClientResult<Option<NumberFor<Block>>> {
+			if hash == Default::default() {
+				Ok(Some(Default::default()))
+			} else {
+				Err(ClientError::Backend("Test error".into()))
+			}
+		}
+
+		fn hash(&self, number: u64) -> ClientResult<Option<Hash>> {
+			if number == 0 {
+				Ok(Some(Default::default()))
+			} else {
+				Err(ClientError::Backend("Test error".into()))
+			}
+		}
+	}
+
+	impl HeaderMetadata<Block> for DummyStorage {
+		type Error = ClientError;
+
+		fn header_metadata(&self, hash: Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
+			self.header(BlockId::hash(hash))?.map(|header| CachedHeaderMetadata::from(&header))
+				.ok_or(ClientError::UnknownBlock("header not found".to_owned()))
+		}
+		fn insert_header_metadata(&self, _hash: Hash, _metadata: CachedHeaderMetadata<Block>) {}
+		fn remove_header_metadata(&self, _hash: Hash) {}
+	}
+
+	impl AuxStore for DummyStorage {
+		fn insert_aux<
+			'a,
+			'b: 'a,
+			'c: 'a,
+			I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
+			D: IntoIterator<Item=&'a &'b [u8]>,
+		>(&self, insert: I, _delete: D) -> ClientResult<()> {
+			for (k, v) in insert.into_iter() {
+				self.aux_store.lock().insert(k.to_vec(), v.to_vec());
+			}
+			Ok(())
+		}
+
+		fn get_aux(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
+			Ok(self.aux_store.lock().get(key).cloned())
+		}
+	}
+
+	impl Storage<Block> for DummyStorage {
+		fn import_header(
+			&self,
+			_header: Header,
+			_cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
+			_state: NewBlockState,
+			_aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		) -> ClientResult<()> {
+			Ok(())
+		}
+
+		fn set_head(&self, _block: BlockId<Block>) -> ClientResult<()> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn finalize_header(&self, _block: BlockId<Block>) -> ClientResult<()> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn last_finalized(&self) -> ClientResult<Hash> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult<Hash> {
+			Err(ClientError::Backend("Test error".into()))
+		}
+
+		fn changes_trie_cht_root(&self, cht_size: u64, block: u64) -> ClientResult<Hash> {
+			cht::block_to_cht_number(cht_size, block)
+				.and_then(|cht_num| self.changes_tries_cht_roots.get(&cht_num))
+				.cloned()
+				.ok_or_else(|| ClientError::Backend(
+					format!("Test error: CHT for block #{} not found", block)
+				).into())
+		}
+
+		fn cache(&self) -> Option<Arc<dyn BlockchainCache<Block>>> {
+			None
+		}
+	}
+}

substrate/client/src/light/call_executor.rs

@@ -0,0 +1,534 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Methods that light client could use to execute runtime calls.
+
+use std::{
+	sync::Arc, panic::UnwindSafe, result, cell::RefCell,
+};
+
+use codec::{Encode, Decode};
+use primitives::{
+	offchain::OffchainExt, H256, Blake2Hasher, convert_hash, NativeOrEncoded,
+	traits::CodeExecutor,
+};
+use sr_primitives::{
+	generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, NumberFor},
+};
+use state_machine::{
+	self, Backend as StateBackend, OverlayedChanges, ExecutionStrategy, create_proof_check_backend,
+	execution_proof_check_on_trie_backend, ExecutionManager, ChangesTrieTransaction, StorageProof,
+	merge_storage_proofs,
+};
+use hash_db::Hasher;
+
+use sr_api::{ProofRecorder, InitializeBlock};
+
+use client_api::{
+	backend::RemoteBackend,
+	error::{Error as ClientError, Result as ClientResult},
+	light::RemoteCallRequest,
+	call_executor::CallExecutor
+};
+use executor::{RuntimeVersion, NativeVersion};
+
+/// Call executor that is able to execute calls only on genesis state.
+///
+/// Trying to execute call on non-genesis state leads to error.
+pub struct GenesisCallExecutor<B, L> {
+	backend: Arc<B>,
+	local: L,
+}
+
+impl<B, L> GenesisCallExecutor<B, L> {
+	/// Create new genesis call executor.
+	pub fn new(backend: Arc<B>, local: L) -> Self {
+		Self { backend, local }
+	}
+}
+
+impl<B, L: Clone> Clone for GenesisCallExecutor<B, L> {
+	fn clone(&self) -> Self {
+		GenesisCallExecutor {
+			backend: self.backend.clone(),
+			local: self.local.clone(),
+		}
+	}
+}
+
+impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
+	GenesisCallExecutor<B, Local>
+	where
+		Block: BlockT<Hash=H256>,
+		B: RemoteBackend<Block, Blake2Hasher>,
+		Local: CallExecutor<Block, Blake2Hasher>,
+{
+	type Error = ClientError;
+
+	fn call(
+		&self,
+		id: &BlockId<Block>,
+		method: &str,
+		call_data: &[u8],
+		strategy: ExecutionStrategy,
+		side_effects_handler: Option<OffchainExt>,
+	) -> ClientResult<Vec<u8>> {
+		match self.backend.is_local_state_available(id) {
+			true => self.local.call(id, method, call_data, strategy, side_effects_handler),
+			false => Err(ClientError::NotAvailableOnLightClient),
+		}
+	}
+
+	fn contextual_call<
+		'a,
+		IB: Fn() -> ClientResult<()>,
+		EM: Fn(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(
+		&self,
+		initialize_block_fn: IB,
+		at: &BlockId<Block>,
+		method: &str,
+		call_data: &[u8],
+		changes: &RefCell<OverlayedChanges>,
+		initialize_block: InitializeBlock<'a, Block>,
+		_manager: ExecutionManager<EM>,
+		native_call: Option<NC>,
+		side_effects_handler: Option<OffchainExt>,
+		recorder: &Option<ProofRecorder<Block>>,
+		enable_keystore: bool,
+	) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
+		// there's no actual way/need to specify native/wasm execution strategy on light node
+		// => we can safely ignore passed values
+
+		match self.backend.is_local_state_available(at) {
+			true => CallExecutor::contextual_call::<
+				_,
+				fn(
+					Result<NativeOrEncoded<R>, Local::Error>,
+					Result<NativeOrEncoded<R>, Local::Error>,
+				) -> Result<NativeOrEncoded<R>, Local::Error>,
+				_,
+				NC
+			>(
+				&self.local,
+				initialize_block_fn,
+				at,
+				method,
+				call_data,
+				changes,
+				initialize_block,
+				ExecutionManager::NativeWhenPossible,
+				native_call,
+				side_effects_handler,
+				recorder,
+				enable_keystore,
+			).map_err(|e| ClientError::Execution(Box::new(e.to_string()))),
+			false => Err(ClientError::NotAvailableOnLightClient),
+		}
+	}
+
+	fn runtime_version(&self, id: &BlockId<Block>) -> ClientResult<RuntimeVersion> {
+		match self.backend.is_local_state_available(id) {
+			true => self.local.runtime_version(id),
+			false => Err(ClientError::NotAvailableOnLightClient),
+		}
+	}
+
+	fn call_at_state<
+		S: StateBackend<Blake2Hasher>,
+		FF: FnOnce(
+			Result<NativeOrEncoded<R>, Self::Error>,
+			Result<NativeOrEncoded<R>, Self::Error>
+		) -> Result<NativeOrEncoded<R>, Self::Error>,
+		R: Encode + Decode + PartialEq,
+		NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+	>(&self,
+		_state: &S,
+		_changes: &mut OverlayedChanges,
+		_method: &str,
+		_call_data: &[u8],
+		_manager: ExecutionManager<FF>,
+		_native_call: Option<NC>,
+		_side_effects_handler: Option<OffchainExt>,
+	) -> ClientResult<(
+		NativeOrEncoded<R>,
+		(S::Transaction, <Blake2Hasher as Hasher>::Out),
+		Option<ChangesTrieTransaction<Blake2Hasher, NumberFor<Block>>>,
+	)> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn prove_at_trie_state<S: state_machine::TrieBackendStorage<Blake2Hasher>>(
+		&self,
+		_state: &state_machine::TrieBackend<S, Blake2Hasher>,
+		_changes: &mut OverlayedChanges,
+		_method: &str,
+		_call_data: &[u8]
+	) -> ClientResult<(Vec<u8>, StorageProof)> {
+		Err(ClientError::NotAvailableOnLightClient)
+	}
+
+	fn native_runtime_version(&self) -> Option<&NativeVersion> {
+		None
+	}
+}
+
+/// Prove contextual execution using given block header in environment.
+///
+/// Method is executed using passed header as environment' current block.
+/// Proof includes both environment preparation proof and method execution proof.
+pub fn prove_execution<Block, S, E>(
+	mut state: S,
+	header: Block::Header,
+	executor: &E,
+	method: &str,
+	call_data: &[u8],
+) -> ClientResult<(Vec<u8>, StorageProof)>
+	where
+		Block: BlockT<Hash=H256>,
+		S: StateBackend<Blake2Hasher>,
+		E: CallExecutor<Block, Blake2Hasher>,
+{
+	let trie_state = state.as_trie_backend()
+		.ok_or_else(|| Box::new(state_machine::ExecutionError::UnableToGenerateProof) as Box<dyn state_machine::Error>)?;
+
+	// prepare execution environment + record preparation proof
+	let mut changes = Default::default();
+	let (_, init_proof) = executor.prove_at_trie_state(
+		trie_state,
+		&mut changes,
+		"Core_initialize_block",
+		&header.encode(),
+	)?;
+
+	// execute method + record execution proof
+	let (result, exec_proof) = executor.prove_at_trie_state(&trie_state, &mut changes, method, call_data)?;
+	let total_proof = merge_storage_proofs(vec![init_proof, exec_proof]);
+
+	Ok((result, total_proof))
+}
+
+/// Check remote contextual execution proof using given backend.
+///
+/// Method is executed using passed header as environment' current block.
+/// Proof should include both environment preparation proof and method execution proof.
+pub fn check_execution_proof<Header, E, H>(
+	executor: &E,
+	request: &RemoteCallRequest<Header>,
+	remote_proof: StorageProof,
+) -> ClientResult<Vec<u8>>
+	where
+		Header: HeaderT,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
+{
+	check_execution_proof_with_make_header::<Header, E, H, _>(
+		executor,
+		request,
+		remote_proof,
+		|header| <Header as HeaderT>::new(
+			*header.number() + One::one(),
+			Default::default(),
+			Default::default(),
+			header.hash(),
+			Default::default(),
+		),
+	)
+}
+
+fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Header) -> Header>(
+	executor: &E,
+	request: &RemoteCallRequest<Header>,
+	remote_proof: StorageProof,
+	make_next_header: MakeNextHeader,
+) -> ClientResult<Vec<u8>>
+	where
+		Header: HeaderT,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
+{
+	let local_state_root = request.header.state_root();
+	let root: H::Out = convert_hash(&local_state_root);
+
+	// prepare execution environment + check preparation proof
+	let mut changes = OverlayedChanges::default();
+	let trie_backend = create_proof_check_backend(root, remote_proof)?;
+	let next_header = make_next_header(&request.header);
+	execution_proof_check_on_trie_backend::<H, _>(
+		&trie_backend,
+		&mut changes,
+		executor,
+		"Core_initialize_block",
+		&next_header.encode(),
+		None,
+	)?;
+
+	// execute method
+	execution_proof_check_on_trie_backend::<H, _>(
+		&trie_backend,
+		&mut changes,
+		executor,
+		&request.method,
+		&request.call_data,
+		None,
+	).map_err(Into::into)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use consensus::BlockOrigin;
+	use test_client::{self, runtime::{Header, Digest, Block}, ClientExt, TestClient};
+	use executor::{NativeExecutor, WasmExecutionMethod};
+	use primitives::Blake2Hasher;
+	use client_api::backend::{Backend, NewBlockState};
+	use crate::in_mem::Backend as InMemBackend;
+
+	struct DummyCallExecutor;
+
+	impl CallExecutor<Block, Blake2Hasher> for DummyCallExecutor {
+		type Error = ClientError;
+
+		fn call(
+			&self,
+			_id: &BlockId<Block>,
+			_method: &str,
+			_call_data: &[u8],
+			_strategy: ExecutionStrategy,
+			_side_effects_handler: Option<OffchainExt>,
+		) -> Result<Vec<u8>, ClientError> {
+			Ok(vec![42])
+		}
+
+		fn contextual_call<
+			'a,
+			IB: Fn() -> ClientResult<()>,
+			EM: Fn(
+				Result<NativeOrEncoded<R>, Self::Error>,
+				Result<NativeOrEncoded<R>, Self::Error>
+			) -> Result<NativeOrEncoded<R>, Self::Error>,
+			R: Encode + Decode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+		>(
+			&self,
+			_initialize_block_fn: IB,
+			_at: &BlockId<Block>,
+			_method: &str,
+			_call_data: &[u8],
+			_changes: &RefCell<OverlayedChanges>,
+			_initialize_block: InitializeBlock<'a, Block>,
+			_execution_manager: ExecutionManager<EM>,
+			_native_call: Option<NC>,
+			_side_effects_handler: Option<OffchainExt>,
+			_proof_recorder: &Option<ProofRecorder<Block>>,
+			_enable_keystore: bool,
+		) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
+			unreachable!()
+		}
+
+		fn runtime_version(&self, _id: &BlockId<Block>) -> Result<RuntimeVersion, ClientError> {
+			unreachable!()
+		}
+
+		fn call_at_state<
+			S: state_machine::Backend<Blake2Hasher>,
+			F: FnOnce(
+				Result<NativeOrEncoded<R>, Self::Error>,
+				Result<NativeOrEncoded<R>, Self::Error>
+			) -> Result<NativeOrEncoded<R>, Self::Error>,
+			R: Encode + Decode + PartialEq,
+			NC: FnOnce() -> result::Result<R, String> + UnwindSafe,
+		>(&self,
+			_state: &S,
+			_overlay: &mut OverlayedChanges,
+			_method: &str,
+			_call_data: &[u8],
+			_manager: ExecutionManager<F>,
+			_native_call: Option<NC>,
+			_side_effects_handler: Option<OffchainExt>,
+		) -> Result<
+			(
+				NativeOrEncoded<R>,
+				(S::Transaction, H256),
+				Option<ChangesTrieTransaction<Blake2Hasher, NumberFor<Block>>>,
+			),
+			ClientError,
+		> {
+			unreachable!()
+		}
+
+		fn prove_at_trie_state<S: state_machine::TrieBackendStorage<Blake2Hasher>>(
+			&self,
+			_trie_state: &state_machine::TrieBackend<S, Blake2Hasher>,
+			_overlay: &mut OverlayedChanges,
+			_method: &str,
+			_call_data: &[u8]
+		) -> Result<(Vec<u8>, StorageProof), ClientError> {
+			unreachable!()
+		}
+
+		fn native_runtime_version(&self) -> Option<&NativeVersion> {
+			unreachable!()
+		}
+	}
+
+	fn local_executor() -> NativeExecutor<test_client::LocalExecutor> {
+		NativeExecutor::new(WasmExecutionMethod::Interpreted, None)
+	}
+
+	#[test]
+	fn execution_proof_is_generated_and_checked() {
+		fn execute(remote_client: &TestClient, at: u64, method: &'static str) -> (Vec<u8>, Vec<u8>) {
+			let remote_block_id = BlockId::Number(at);
+			let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+
+			// 'fetch' execution proof from remote node
+			let (remote_result, remote_execution_proof) = remote_client.execution_proof(
+				&remote_block_id,
+				method,
+				&[]
+			).unwrap();
+
+			// check remote execution proof locally
+			let local_result = check_execution_proof::<_, _, Blake2Hasher>(
+				&local_executor(),
+				&RemoteCallRequest {
+					block: test_client::runtime::Hash::default(),
+					header: remote_header,
+					method: method.into(),
+					call_data: vec![],
+					retry_count: None,
+				},
+				remote_execution_proof,
+			).unwrap();
+
+			(remote_result, local_result)
+		}
+
+		fn execute_with_proof_failure(remote_client: &TestClient, at: u64, method: &'static str) {
+			let remote_block_id = BlockId::Number(at);
+			let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+
+			// 'fetch' execution proof from remote node
+			let (_, remote_execution_proof) = remote_client.execution_proof(
+				&remote_block_id,
+				method,
+				&[]
+			).unwrap();
+
+			// check remote execution proof locally
+			let execution_result = check_execution_proof_with_make_header::<_, _, Blake2Hasher, _>(
+				&local_executor(),
+				&RemoteCallRequest {
+					block: test_client::runtime::Hash::default(),
+					header: remote_header,
+					method: method.into(),
+					call_data: vec![],
+					retry_count: None,
+				},
+				remote_execution_proof,
+				|header| <Header as HeaderT>::new(
+					at + 1,
+					Default::default(),
+					Default::default(),
+					header.hash(),
+					header.digest().clone(), // this makes next header wrong
+				),
+			);
+			match execution_result {
+				Err(client_api::error::Error::Execution(_)) => (),
+				_ => panic!("Unexpected execution result: {:?}", execution_result),
+			}
+		}
+
+		// prepare remote client
+		let remote_client = test_client::new();
+		for i in 1u32..3u32 {
+			let mut digest = Digest::default();
+			digest.push(sr_primitives::generic::DigestItem::Other::<H256>(i.to_le_bytes().to_vec()));
+			remote_client.import_justified(
+				BlockOrigin::Own,
+				remote_client.new_block(digest).unwrap().bake().unwrap(),
+				Default::default(),
+			).unwrap();
+		}
+
+		// check method that doesn't requires environment
+		let (remote, local) = execute(&remote_client, 0, "Core_version");
+		assert_eq!(remote, local);
+
+		let (remote, local) = execute(&remote_client, 2, "Core_version");
+		assert_eq!(remote, local);
+
+		// check method that requires environment
+		let (_, block) = execute(&remote_client, 0, "BlockBuilder_finalize_block");
+		let local_block: Header = Decode::decode(&mut &block[..]).unwrap();
+		assert_eq!(local_block.number, 1);
+
+		let (_, block) = execute(&remote_client, 2, "BlockBuilder_finalize_block");
+		let local_block: Header = Decode::decode(&mut &block[..]).unwrap();
+		assert_eq!(local_block.number, 3);
+
+		// check that proof check doesn't panic even if proof is incorrect AND no panic handler is set
+		execute_with_proof_failure(&remote_client, 2, "Core_version");
+
+		// check that proof check doesn't panic even if proof is incorrect AND panic handler is set
+		panic_handler::set("TEST", "1.2.3");
+		execute_with_proof_failure(&remote_client, 2, "Core_version");
+	}
+
+	#[test]
+	fn code_is_executed_at_genesis_only() {
+		let backend = Arc::new(InMemBackend::<Block, Blake2Hasher>::new());
+		let def = H256::default();
+		let header0 = test_client::runtime::Header::new(0, def, def, def, Default::default());
+		let hash0 = header0.hash();
+		let header1 = test_client::runtime::Header::new(1, def, def, hash0, Default::default());
+		let hash1 = header1.hash();
+		backend.blockchain().insert(hash0, header0, None, None, NewBlockState::Final).unwrap();
+		backend.blockchain().insert(hash1, header1, None, None, NewBlockState::Final).unwrap();
+
+		let genesis_executor = GenesisCallExecutor::new(backend, DummyCallExecutor);
+		assert_eq!(
+			genesis_executor.call(
+				&BlockId::Number(0),
+				"test_method",
+				&[],
+				ExecutionStrategy::NativeElseWasm,
+				None,
+			).unwrap(),
+			vec![42],
+		);
+
+		let call_on_unavailable = genesis_executor.call(
+			&BlockId::Number(1),
+			"test_method",
+			&[],
+			ExecutionStrategy::NativeElseWasm,
+			None,
+		);
+
+		match call_on_unavailable {
+			Err(ClientError::NotAvailableOnLightClient) => (),
+			_ => unreachable!("unexpected result: {:?}", call_on_unavailable),
+		}
+	}
+}

substrate/client/src/light/fetcher.rs

@@ -0,0 +1,783 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client data fetcher. Fetches requested data from remote full nodes.
+
+use std::sync::Arc;
+use std::collections::{BTreeMap, HashMap};
+use std::marker::PhantomData;
+
+use hash_db::{HashDB, Hasher, EMPTY_PREFIX};
+use codec::{Decode, Encode};
+use primitives::{convert_hash, traits::CodeExecutor, H256};
+use sr_primitives::traits::{
+	Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor,
+	SimpleArithmetic, CheckedConversion, Zero,
+};
+use state_machine::{
+	ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange,
+	TrieBackend, read_proof_check, key_changes_proof_check, create_proof_check_backend_storage,
+	read_child_proof_check,
+};
+pub use state_machine::StorageProof;
+
+use crate::cht;
+pub use client_api::{
+	error::{
+		Error as ClientError, Result as ClientResult
+	},
+	light::{
+		RemoteCallRequest, RemoteHeaderRequest, RemoteReadRequest, RemoteReadChildRequest,
+		RemoteChangesRequest, ChangesProof, RemoteBodyRequest, Fetcher, FetchChecker,
+		Storage as BlockchainStorage,
+	},
+};
+use crate::light::blockchain::{Blockchain};
+use crate::light::call_executor::check_execution_proof;
+
+/// Remote data checker.
+pub struct LightDataChecker<E, H, B: BlockT, S: BlockchainStorage<B>> {
+	blockchain: Arc<Blockchain<S>>,
+	executor: E,
+	_hasher: PhantomData<(B, H)>,
+}
+
+impl<E, H, B: BlockT, S: BlockchainStorage<B>> LightDataChecker<E, H, B, S> {
+	/// Create new light data checker.
+	pub fn new(blockchain: Arc<Blockchain<S>>, executor: E) -> Self {
+		Self {
+			blockchain, executor, _hasher: PhantomData
+		}
+	}
+
+	/// Check remote changes query proof assuming that CHT-s are of given size.
+	fn check_changes_proof_with_cht_size(
+		&self,
+		request: &RemoteChangesRequest<B::Header>,
+		remote_proof: ChangesProof<B::Header>,
+		cht_size: NumberFor<B>,
+	) -> ClientResult<Vec<(NumberFor<B>, u32)>>
+		where
+			H: Hasher,
+			H::Out: Ord,
+	{
+		// since we need roots of all changes tries for the range begin..max
+		// => remote node can't use max block greater that one that we have passed
+		if remote_proof.max_block > request.max_block.0 || remote_proof.max_block < request.last_block.0 {
+			return Err(ClientError::ChangesTrieAccessFailed(format!(
+				"Invalid max_block used by the remote node: {}. Local: {}..{}..{}",
+				remote_proof.max_block, request.first_block.0, request.last_block.0, request.max_block.0,
+			)).into());
+		}
+
+		// check if remote node has responded with extra changes trie roots proofs
+		// all changes tries roots must be in range [request.first_block.0; request.tries_roots.0)
+		let is_extra_first_root = remote_proof.roots.keys().next()
+			.map(|first_root| *first_root < request.first_block.0
+				|| *first_root >= request.tries_roots.0)
+			.unwrap_or(false);
+		let is_extra_last_root = remote_proof.roots.keys().next_back()
+			.map(|last_root| *last_root >= request.tries_roots.0)
+			.unwrap_or(false);
+		if is_extra_first_root || is_extra_last_root {
+			return Err(ClientError::ChangesTrieAccessFailed(format!(
+				"Extra changes tries roots proofs provided by the remote node: [{:?}..{:?}]. Expected in range: [{}; {})",
+				remote_proof.roots.keys().next(), remote_proof.roots.keys().next_back(),
+				request.first_block.0, request.tries_roots.0,
+			)).into());
+		}
+
+		// if request has been composed when some required headers were already pruned
+		// => remote node has sent us CHT-based proof of required changes tries roots
+		// => check that this proof is correct before proceeding with changes proof
+		let remote_max_block = remote_proof.max_block;
+		let remote_roots = remote_proof.roots;
+		let remote_roots_proof = remote_proof.roots_proof;
+		let remote_proof = remote_proof.proof;
+		if !remote_roots.is_empty() {
+			self.check_changes_tries_proof(
+				cht_size,
+				&remote_roots,
+				remote_roots_proof,
+			)?;
+		}
+
+		// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
+		let changes_trie_config_range = ChangesTrieConfigurationRange {
+			config: &request.changes_trie_config,
+			zero: Zero::zero(),
+			end: None,
+		};
+
+		// and now check the key changes proof + get the changes
+		key_changes_proof_check::<H, _>(
+			changes_trie_config_range,
+			&RootsStorage {
+				roots: (request.tries_roots.0, &request.tries_roots.2),
+				prev_roots: remote_roots,
+			},
+			remote_proof,
+			request.first_block.0,
+			&ChangesTrieAnchorBlockId {
+				hash: convert_hash(&request.last_block.1),
+				number: request.last_block.0,
+			},
+			remote_max_block,
+			request.storage_key.as_ref().map(Vec::as_slice),
+			&request.key)
+		.map_err(|err| ClientError::ChangesTrieAccessFailed(err))
+	}
+
+	/// Check CHT-based proof for changes tries roots.
+	fn check_changes_tries_proof(
+		&self,
+		cht_size: NumberFor<B>,
+		remote_roots: &BTreeMap<NumberFor<B>, B::Hash>,
+		remote_roots_proof: StorageProof,
+	) -> ClientResult<()>
+		where
+			H: Hasher,
+			H::Out: Ord,
+	{
+		// all the checks are sharing the same storage
+		let storage = create_proof_check_backend_storage(remote_roots_proof);
+
+		// we remote_roots.keys() are sorted => we can use this to group changes tries roots
+		// that are belongs to the same CHT
+		let blocks = remote_roots.keys().cloned();
+		cht::for_each_cht_group::<B::Header, _, _, _>(cht_size, blocks, |mut storage, _, cht_blocks| {
+			// get local changes trie CHT root for given CHT
+			// it should be there, because it is never pruned AND request has been composed
+			// when required header has been pruned (=> replaced with CHT)
+			let first_block = cht_blocks.first().cloned()
+				.expect("for_each_cht_group never calls callback with empty groups");
+			let local_cht_root = self.blockchain.storage().changes_trie_cht_root(cht_size, first_block)?;
+
+			// check changes trie root for every block within CHT range
+			for block in cht_blocks {
+				// check if the proofs storage contains the root
+				// normally this happens in when the proving backend is created, but since
+				// we share the storage for multiple checks, do it here
+				let mut cht_root = H::Out::default();
+				cht_root.as_mut().copy_from_slice(local_cht_root.as_ref());
+				if !storage.contains(&cht_root, EMPTY_PREFIX) {
+					return Err(ClientError::InvalidCHTProof.into());
+				}
+
+				// check proof for single changes trie root
+				let proving_backend = TrieBackend::new(storage, cht_root);
+				let remote_changes_trie_root = remote_roots[&block];
+				cht::check_proof_on_proving_backend::<B::Header, H>(
+					local_cht_root,
+					block,
+					remote_changes_trie_root,
+					&proving_backend)?;
+
+				// and return the storage to use in following checks
+				storage = proving_backend.into_storage();
+			}
+
+			Ok(storage)
+		}, storage)
+	}
+}
+
+impl<E, Block, H, S> FetchChecker<Block> for LightDataChecker<E, H, Block, S>
+	where
+		Block: BlockT,
+		E: CodeExecutor,
+		H: Hasher<Out=H256>,
+		S: BlockchainStorage<Block>,
+{
+	fn check_header_proof(
+		&self,
+		request: &RemoteHeaderRequest<Block::Header>,
+		remote_header: Option<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<Block::Header> {
+		let remote_header = remote_header.ok_or_else(||
+			ClientError::from(ClientError::InvalidCHTProof))?;
+		let remote_header_hash = remote_header.hash();
+		cht::check_proof::<Block::Header, H>(
+			request.cht_root,
+			request.block,
+			remote_header_hash,
+			remote_proof)
+			.map(|_| remote_header)
+	}
+
+	fn check_read_proof(
+		&self,
+		request: &RemoteReadRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>> {
+		read_proof_check::<H, _>(
+			convert_hash(request.header.state_root()),
+			remote_proof,
+			request.keys.iter(),
+		).map_err(Into::into)
+	}
+
+	fn check_read_child_proof(
+		&self,
+		request: &RemoteReadChildRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<HashMap<Vec<u8>, Option<Vec<u8>>>> {
+		read_child_proof_check::<H, _>(
+			convert_hash(request.header.state_root()),
+			remote_proof,
+			&request.storage_key,
+			request.keys.iter(),
+		).map_err(Into::into)
+	}
+
+	fn check_execution_proof(
+		&self,
+		request: &RemoteCallRequest<Block::Header>,
+		remote_proof: StorageProof,
+	) -> ClientResult<Vec<u8>> {
+		check_execution_proof::<_, _, H>(&self.executor, request, remote_proof)
+	}
+
+	fn check_changes_proof(
+		&self,
+		request: &RemoteChangesRequest<Block::Header>,
+		remote_proof: ChangesProof<Block::Header>
+	) -> ClientResult<Vec<(NumberFor<Block>, u32)>> {
+		self.check_changes_proof_with_cht_size(request, remote_proof, cht::size())
+	}
+
+	fn check_body_proof(
+		&self,
+		request: &RemoteBodyRequest<Block::Header>,
+		body: Vec<Block::Extrinsic>
+	) -> ClientResult<Vec<Block::Extrinsic>> {
+		// TODO: #2621
+		let	extrinsics_root = HashFor::<Block>::ordered_trie_root(
+			body.iter().map(Encode::encode).collect(),
+		);
+		if *request.header.extrinsics_root() == extrinsics_root {
+			Ok(body)
+		} else {
+			Err(format!("RemoteBodyRequest: invalid extrinsics root expected: {} but got {}",
+				*request.header.extrinsics_root(),
+				extrinsics_root,
+			).into())
+		}
+
+	}
+}
+
+/// A view of BTreeMap<Number, Hash> as a changes trie roots storage.
+struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> {
+	roots: (Number, &'a [Hash]),
+	prev_roots: BTreeMap<Number, Hash>,
+}
+
+impl<'a, H, Number, Hash> ChangesTrieRootsStorage<H, Number> for RootsStorage<'a, Number, Hash>
+	where
+		H: Hasher,
+		Number: ::std::fmt::Display + ::std::hash::Hash + Clone + SimpleArithmetic + Encode + Decode + Send + Sync + 'static,
+		Hash: 'a + Send + Sync + Clone + AsRef<[u8]>,
+{
+	fn build_anchor(
+		&self,
+		_hash: H::Out,
+	) -> Result<state_machine::ChangesTrieAnchorBlockId<H::Out, Number>, String> {
+		Err("build_anchor is only called when building block".into())
+	}
+
+	fn root(
+		&self,
+		_anchor: &ChangesTrieAnchorBlockId<H::Out, Number>,
+		block: Number,
+	) -> Result<Option<H::Out>, String> {
+		// we can't ask for roots from parallel forks here => ignore anchor
+		let root = if block < self.roots.0 {
+			self.prev_roots.get(&Number::unique_saturated_from(block)).cloned()
+		} else {
+			let index: Option<usize> = block.checked_sub(&self.roots.0).and_then(|index| index.checked_into());
+			match index {
+				Some(index) => self.roots.1.get(index as usize).cloned(),
+				None => None,
+			}
+		};
+
+		Ok(root.map(|root| {
+			let mut hasher_root: H::Out = Default::default();
+			hasher_root.as_mut().copy_from_slice(root.as_ref());
+			hasher_root
+		}))
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use codec::Decode;
+	use crate::client::tests::prepare_client_with_key_changes;
+	use executor::{NativeExecutor, WasmExecutionMethod};
+	use client_api::{
+		backend::NewBlockState,
+		error::Error as ClientError,
+	};
+	use test_client::{
+		self, ClientExt, blockchain::HeaderBackend, AccountKeyring,
+		runtime::{self, Hash, Block, Header, Extrinsic}
+	};
+	use consensus::BlockOrigin;
+
+	use crate::in_mem::{Blockchain as InMemoryBlockchain};
+	use crate::light::fetcher::{FetchChecker, LightDataChecker, RemoteHeaderRequest};
+	use crate::light::blockchain::tests::{DummyStorage, DummyBlockchain};
+	use primitives::{blake2_256, Blake2Hasher, H256};
+	use primitives::storage::{well_known_keys, StorageKey};
+	use sr_primitives::generic::BlockId;
+	use state_machine::Backend;
+	use super::*;
+
+	type TestChecker = LightDataChecker<
+		NativeExecutor<test_client::LocalExecutor>,
+		Blake2Hasher,
+		Block,
+		DummyStorage,
+	>;
+
+	fn local_executor() -> NativeExecutor<test_client::LocalExecutor> {
+		NativeExecutor::new(WasmExecutionMethod::Interpreted, None)
+	}
+
+	fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap();
+		let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+		remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap()
+			.storage_root(::std::iter::empty()).0.into();
+
+		// 'fetch' read proof from remote node
+		let heap_pages = remote_client.storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec()))
+			.unwrap()
+			.and_then(|v| Decode::decode(&mut &v.0[..]).ok()).unwrap();
+		let remote_read_proof = remote_client.read_proof(
+			&remote_block_id,
+			&[well_known_keys::HEAP_PAGES],
+		).unwrap();
+
+		// check remote read proof locally
+		let local_storage = InMemoryBlockchain::<Block>::new();
+		local_storage.insert(
+			remote_block_hash,
+			remote_block_header.clone(),
+			None,
+			None,
+			NewBlockState::Final,
+		).unwrap();
+		let local_checker = LightDataChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor()
+		);
+		(local_checker, remote_block_header, remote_read_proof, heap_pages)
+	}
+
+	fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, Vec<u8>) {
+		use test_client::DefaultTestClientBuilderExt;
+		use test_client::TestClientBuilderExt;
+		// prepare remote client
+		let remote_client = test_client::TestClientBuilder::new()
+			.add_extra_child_storage(b":child_storage:default:child1".to_vec(), b"key1".to_vec(), b"value1".to_vec())
+			.build();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap();
+		let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap();
+		remote_block_header.state_root = remote_client.state_at(&remote_block_id).unwrap()
+			.storage_root(::std::iter::empty()).0.into();
+
+		// 'fetch' child read proof from remote node
+		let child_value = remote_client.child_storage(
+			&remote_block_id,
+			&StorageKey(b":child_storage:default:child1".to_vec()),
+			&StorageKey(b"key1".to_vec()),
+		).unwrap().unwrap().0;
+		assert_eq!(b"value1"[..], child_value[..]);
+		let remote_read_proof = remote_client.read_child_proof(
+			&remote_block_id,
+			b":child_storage:default:child1",
+			&[b"key1"],
+		).unwrap();
+
+		// check locally
+		let local_storage = InMemoryBlockchain::<Block>::new();
+		local_storage.insert(
+			remote_block_hash,
+			remote_block_header.clone(),
+			None,
+			None,
+			NewBlockState::Final,
+		).unwrap();
+		let local_checker = LightDataChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+		(local_checker, remote_block_header, remote_read_proof, child_value)
+	}
+
+	fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Header, StorageProof) {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let mut local_headers_hashes = Vec::new();
+		for i in 0..4 {
+			let builder = remote_client.new_block(Default::default()).unwrap();
+			remote_client.import(BlockOrigin::Own, builder.bake().unwrap()).unwrap();
+			local_headers_hashes.push(remote_client.block_hash(i + 1)
+				.map_err(|_| ClientError::Backend("TestError".into())));
+		}
+
+		// 'fetch' header proof from remote node
+		let remote_block_id = BlockId::Number(1);
+		let (remote_block_header, remote_header_proof) = remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap();
+
+		// check remote read proof locally
+		let local_storage = InMemoryBlockchain::<Block>::new();
+		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(4, 0, local_headers_hashes).unwrap();
+		if insert_cht {
+			local_storage.insert_cht_root(1, local_cht_root);
+		}
+		let local_checker = LightDataChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+		(local_checker, local_cht_root, remote_block_header, remote_header_proof)
+	}
+
+	fn header_with_computed_extrinsics_root(extrinsics: Vec<Extrinsic>) -> Header {
+		use trie::{TrieConfiguration, trie_types::Layout};
+		let iter = extrinsics.iter().map(Encode::encode);
+		let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter);
+
+		// only care about `extrinsics_root`
+		Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default())
+	}
+
+	#[test]
+	fn storage_read_proof_is_generated_and_checked() {
+		let (local_checker, remote_block_header, remote_read_proof, heap_pages) = prepare_for_read_proof_check();
+		assert_eq!((&local_checker as &dyn FetchChecker<Block>).check_read_proof(&RemoteReadRequest::<Header> {
+			block: remote_block_header.hash(),
+			header: remote_block_header,
+			keys: vec![well_known_keys::HEAP_PAGES.to_vec()],
+			retry_count: None,
+		}, remote_read_proof).unwrap().remove(well_known_keys::HEAP_PAGES).unwrap().unwrap()[0], heap_pages as u8);
+	}
+
+	#[test]
+	fn storage_child_read_proof_is_generated_and_checked() {
+		let (
+			local_checker,
+			remote_block_header,
+			remote_read_proof,
+			result,
+		) = prepare_for_read_child_proof_check();
+		assert_eq!((&local_checker as &dyn FetchChecker<Block>).check_read_child_proof(
+			&RemoteReadChildRequest::<Header> {
+				block: remote_block_header.hash(),
+				header: remote_block_header,
+				storage_key: b":child_storage:default:child1".to_vec(),
+				keys: vec![b"key1".to_vec()],
+				retry_count: None,
+			},
+			remote_read_proof
+		).unwrap().remove(b"key1".as_ref()).unwrap().unwrap(), result);
+	}
+
+	#[test]
+	fn header_proof_is_generated_and_checked() {
+		let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		assert_eq!((&local_checker as &dyn FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: local_cht_root,
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).unwrap(), remote_block_header);
+	}
+
+	#[test]
+	fn check_header_proof_fails_if_cht_root_is_invalid() {
+		let (local_checker, _, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		remote_block_header.number = 100;
+		assert!((&local_checker as &dyn FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: Default::default(),
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).is_err());
+	}
+
+	#[test]
+	fn check_header_proof_fails_if_invalid_header_provided() {
+		let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = prepare_for_header_proof_check(true);
+		remote_block_header.number = 100;
+		assert!((&local_checker as &dyn FetchChecker<Block>).check_header_proof(&RemoteHeaderRequest::<Header> {
+			cht_root: local_cht_root,
+			block: 1,
+			retry_count: None,
+		}, Some(remote_block_header.clone()), remote_header_proof).is_err());
+	}
+
+	#[test]
+	fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() {
+		let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes();
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+		let local_checker = &local_checker as &dyn FetchChecker<Block>;
+		let max = remote_client.info().chain.best_number;
+		let max_hash = remote_client.info().chain.best_hash;
+
+		for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() {
+			let begin_hash = remote_client.block_hash(begin).unwrap().unwrap();
+			let end_hash = remote_client.block_hash(end).unwrap().unwrap();
+
+			// 'fetch' changes proof from remote node
+			let key = StorageKey(key);
+			let remote_proof = remote_client.key_changes_proof(
+				begin_hash, end_hash, begin_hash, max_hash, None, &key
+			).unwrap();
+
+			// check proof on local client
+			let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
+			let request = RemoteChangesRequest::<Header> {
+				changes_trie_config: runtime::changes_trie_config(),
+				first_block: (begin, begin_hash),
+				last_block: (end, end_hash),
+				max_block: (max, max_hash),
+				tries_roots: (begin, begin_hash, local_roots_range),
+				key: key.0,
+				storage_key: None,
+				retry_count: None,
+			};
+			let local_result = local_checker.check_changes_proof(&request, ChangesProof {
+				max_block: remote_proof.max_block,
+				proof: remote_proof.proof,
+				roots: remote_proof.roots,
+				roots_proof: remote_proof.roots_proof,
+			}).unwrap();
+
+			// ..and ensure that result is the same as on remote node
+			match local_result == expected_result {
+				true => (),
+				false => panic!(format!("Failed test {}: local = {:?}, expected = {:?}",
+					index, local_result, expected_result)),
+			}
+		}
+	}
+
+	#[test]
+	fn changes_proof_is_generated_and_checked_when_headers_are_pruned() {
+		// we're testing this test case here:
+		// (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]),
+		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
+		let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let dave = StorageKey(dave);
+
+		// 'fetch' changes proof from remote node:
+		// we're fetching changes for range b1..b4
+		// we do not know changes trie roots before b3 (i.e. we only know b3+b4)
+		// but we have changes trie CHT root for b1...b4
+		let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap();
+		let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap();
+		let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap();
+		let remote_proof = remote_client.key_changes_proof_with_cht_size(
+			b1, b4, b3, b4, None, &dave, 4
+		).unwrap();
+
+		// prepare local checker, having a root of changes trie CHT#0
+		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap();
+		let mut local_storage = DummyStorage::new();
+		local_storage.changes_tries_cht_roots.insert(0, local_cht_root);
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(local_storage)),
+			local_executor(),
+		);
+
+		// check proof on local client
+		let request = RemoteChangesRequest::<Header> {
+			changes_trie_config: runtime::changes_trie_config(),
+			first_block: (1, b1),
+			last_block: (4, b4),
+			max_block: (4, b4),
+			tries_roots: (3, b3, vec![remote_roots[2].clone(), remote_roots[3].clone()]),
+			storage_key: None,
+			key: dave.0,
+			retry_count: None,
+		};
+		let local_result = local_checker.check_changes_proof_with_cht_size(&request, ChangesProof {
+			max_block: remote_proof.max_block,
+			proof: remote_proof.proof,
+			roots: remote_proof.roots,
+			roots_proof: remote_proof.roots_proof,
+		}, 4).unwrap();
+
+		assert_eq!(local_result, vec![(4, 0), (1, 1), (1, 0)]);
+	}
+
+	#[test]
+	fn check_changes_proof_fails_if_proof_is_wrong() {
+		let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes();
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+		let local_checker = &local_checker as &dyn FetchChecker<Block>;
+		let max = remote_client.info().chain.best_number;
+		let max_hash = remote_client.info().chain.best_hash;
+
+		let (begin, end, key, _) = test_cases[0].clone();
+		let begin_hash = remote_client.block_hash(begin).unwrap().unwrap();
+		let end_hash = remote_client.block_hash(end).unwrap().unwrap();
+
+		// 'fetch' changes proof from remote node
+		let key = StorageKey(key);
+		let remote_proof = remote_client.key_changes_proof(
+			begin_hash, end_hash, begin_hash, max_hash, None, &key).unwrap();
+
+		let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec();
+		let request = RemoteChangesRequest::<Header> {
+			changes_trie_config: runtime::changes_trie_config(),
+			first_block: (begin, begin_hash),
+			last_block: (end, end_hash),
+			max_block: (max, max_hash),
+			tries_roots: (begin, begin_hash, local_roots_range.clone()),
+			storage_key: None,
+			key: key.0,
+			retry_count: None,
+		};
+
+		// check proof on local client using max from the future
+		assert!(local_checker.check_changes_proof(&request, ChangesProof {
+			max_block: remote_proof.max_block + 1,
+			proof: remote_proof.proof.clone(),
+			roots: remote_proof.roots.clone(),
+			roots_proof: remote_proof.roots_proof.clone(),
+		}).is_err());
+
+		// check proof on local client using broken proof
+		assert!(local_checker.check_changes_proof(&request, ChangesProof {
+			max_block: remote_proof.max_block,
+			proof: local_roots_range.clone().into_iter().map(|v| v.as_ref().to_vec()).collect(),
+			roots: remote_proof.roots,
+			roots_proof: remote_proof.roots_proof,
+		}).is_err());
+
+		// extra roots proofs are provided
+		assert!(local_checker.check_changes_proof(&request, ChangesProof {
+			max_block: remote_proof.max_block,
+			proof: remote_proof.proof.clone(),
+			roots: vec![(begin - 1, Default::default())].into_iter().collect(),
+			roots_proof: StorageProof::empty(),
+		}).is_err());
+		assert!(local_checker.check_changes_proof(&request, ChangesProof {
+			max_block: remote_proof.max_block,
+			proof: remote_proof.proof.clone(),
+			roots: vec![(end + 1, Default::default())].into_iter().collect(),
+			roots_proof: StorageProof::empty(),
+		}).is_err());
+	}
+
+	#[test]
+	fn check_changes_tries_proof_fails_if_proof_is_wrong() {
+		// we're testing this test case here:
+		// (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]),
+		let (remote_client, remote_roots, _) = prepare_client_with_key_changes();
+		let local_cht_root = cht::compute_root::<Header, Blake2Hasher, _>(
+			4, 0, remote_roots.iter().cloned().map(|ct| Ok(Some(ct)))).unwrap();
+		let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec();
+		let dave = StorageKey(dave);
+
+		// 'fetch' changes proof from remote node:
+		// we're fetching changes for range b1..b4
+		// we do not know changes trie roots before b3 (i.e. we only know b3+b4)
+		// but we have changes trie CHT root for b1...b4
+		let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap();
+		let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap();
+		let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap();
+		let remote_proof = remote_client.key_changes_proof_with_cht_size(
+			b1, b4, b3, b4, None, &dave, 4
+		).unwrap();
+
+		// fails when changes trie CHT is missing from the local db
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+		assert!(local_checker.check_changes_tries_proof(4, &remote_proof.roots,
+			remote_proof.roots_proof.clone()).is_err());
+
+		// fails when proof is broken
+		let mut local_storage = DummyStorage::new();
+		local_storage.changes_tries_cht_roots.insert(0, local_cht_root);
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(local_storage)),
+			local_executor(),
+		);
+		let result = local_checker.check_changes_tries_proof(
+			4, &remote_proof.roots, StorageProof::empty()
+		);
+		assert!(result.is_err());
+	}
+
+	#[test]
+	fn check_body_proof_faulty() {
+		let header = header_with_computed_extrinsics_root(
+			vec![Extrinsic::IncludeData(vec![1, 2, 3, 4])]
+		);
+		let block = Block::new(header.clone(), Vec::new());
+
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+
+		let body_request = RemoteBodyRequest {
+			header: header.clone(),
+			retry_count: None,
+		};
+
+		assert!(
+			local_checker.check_body_proof(&body_request, block.extrinsics).is_err(),
+			"vec![1, 2, 3, 4] != vec![]"
+		);
+	}
+
+	#[test]
+	fn check_body_proof_of_same_data_should_succeed() {
+		let extrinsics = vec![Extrinsic::IncludeData(vec![1, 2, 3, 4, 5, 6, 7, 8, 255])];
+
+		let header = header_with_computed_extrinsics_root(extrinsics.clone());
+		let block = Block::new(header.clone(), extrinsics);
+
+		let local_checker = TestChecker::new(
+			Arc::new(DummyBlockchain::new(DummyStorage::new())),
+			local_executor(),
+		);
+
+		let body_request = RemoteBodyRequest {
+			header: header.clone(),
+			retry_count: None,
+		};
+
+		assert!(local_checker.check_body_proof(&body_request, block.extrinsics).is_ok());
+	}
+}

substrate/client/src/light/mod.rs

@@ -0,0 +1,85 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Light client components.
+
+pub mod backend;
+pub mod blockchain;
+pub mod call_executor;
+pub mod fetcher;
+
+use std::sync::Arc;
+
+use executor::RuntimeInfo;
+use primitives::{H256, Blake2Hasher, traits::CodeExecutor};
+use sr_primitives::BuildStorage;
+use sr_primitives::traits::Block as BlockT;
+
+use crate::call_executor::LocalCallExecutor;
+use crate::client::Client;
+use client_api::{
+	error::Result as ClientResult,
+	light::Storage as BlockchainStorage,
+};
+use crate::light::backend::Backend;
+use crate::light::blockchain::Blockchain;
+use crate::light::call_executor::GenesisCallExecutor;
+use crate::light::fetcher::LightDataChecker;
+
+/// Create an instance of light client blockchain backend.
+pub fn new_light_blockchain<B: BlockT, S: BlockchainStorage<B>>(storage: S) -> Arc<Blockchain<S>> {
+	Arc::new(Blockchain::new(storage))
+}
+
+/// Create an instance of light client backend.
+pub fn new_light_backend<B, S>(blockchain: Arc<Blockchain<S>>) -> Arc<Backend<S, Blake2Hasher>>
+	where
+		B: BlockT,
+		S: BlockchainStorage<B>,
+{
+	Arc::new(Backend::new(blockchain))
+}
+
+/// Create an instance of light client.
+pub fn new_light<B, S, GS, RA, E>(
+	backend: Arc<Backend<S, Blake2Hasher>>,
+	genesis_storage: GS,
+	code_executor: E,
+) -> ClientResult<Client<Backend<S, Blake2Hasher>, GenesisCallExecutor<
+	Backend<S, Blake2Hasher>,
+	LocalCallExecutor<Backend<S, Blake2Hasher>, E>
+>, B, RA>>
+	where
+		B: BlockT<Hash=H256>,
+		S: BlockchainStorage<B> + 'static,
+		GS: BuildStorage,
+		E: CodeExecutor + RuntimeInfo,
+{
+	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, None);
+	let executor = GenesisCallExecutor::new(backend.clone(), local_executor);
+	Client::new(backend, executor, genesis_storage, Default::default(), Default::default())
+}
+
+/// Create an instance of fetch data checker.
+pub fn new_fetch_checker<E, B: BlockT, S: BlockchainStorage<B>>(
+	blockchain: Arc<Blockchain<S>>,
+	executor: E,
+) -> LightDataChecker<E, Blake2Hasher, B, S>
+	where
+		E: CodeExecutor,
+{
+	LightDataChecker::new(blockchain, executor)
+}